jschoubben
cb353f9881
Run 11 failed with node2's daemon.json never written and nothing to say whether the declaration lacked the trust or never applied. The dump now answers that, and the push output is printed so a compose that refused is visible in the run log.
325 lines
17 KiB
TypeScript
325 lines
17 KiB
TypeScript
/**
|
|
* THE NO-FAKE MULTI-NODE GATE: two machines, everything built by the mesh itself.
|
|
*
|
|
* The rewrite-based multi-node beds pre-stock runtime images and rewrite each manifest's
|
|
* placeholder digest to whatever they stocked — bed code doing the builder's job, which means the
|
|
* builder's job was never under test. This bed removes the shortcut. The scenario names NO images;
|
|
* the committed manifests are registered VERBATIM; the only thing that ever turns
|
|
* `mesh-runtime-<m>@sha256:0000…` (or an `artifact:` reference) into a real digest is the mesh's
|
|
* own builder, exactly as in production.
|
|
*
|
|
* What it proves, end to end:
|
|
* 1. The installer raises `anchor` into a mesh of one — building the control plane (ADR 0073),
|
|
* standing up the registry and the builder.
|
|
* 2. The mesh BUILDS the shared base, then postgres and lavinmq, from the forge — and adopts the
|
|
* foundation's own store and broker as those modules (ADR 0078), the store's genesis superuser
|
|
* carried in through `secret accept` (the same act as hq phase3 deliverSuperuser).
|
|
* 3. `node2` joins, and its consumers are BUILT and delivered the same way: `amqp-ping` opens the
|
|
* one broker and `letta` gets a database on the one store — both across the overlay, admitted
|
|
* by the firewall the nftables module derives (issues 055/056/057 in one bed).
|
|
*
|
|
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BOOTSTRAP_BINARY=.../mesh-bootstrap
|
|
* MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock (the TEMPLATE)
|
|
* MESH_LAB_CATALOG=.../mesh-catalog/modules
|
|
* MESH_LAB_SOURCE=git://<forge>/mesh-controller.git MESH_LAB_SOURCE_REF=<commit>
|
|
* MESH_LAB_BUILD_REF=<branch or commit for module builds, default main>
|
|
*/
|
|
import { test, before, after } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { existsSync, readFileSync } from "node:fs";
|
|
import { resolve } from "node:path";
|
|
import { loadScenario } from "../../src/declaration/parse.ts";
|
|
import { raise } from "../../src/lifecycle/raise.ts";
|
|
import { destroy, exec, push } from "../../src/lifecycle/operate.ts";
|
|
import { bootstrapBinaryPath, hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
|
import { labIsUsable, destroyAll, foundationBundle } from "./harness.ts";
|
|
import { genesis, type GenesisResult } from "./genesis.ts";
|
|
import { keep, ready, returnTo } from "../../src/warm.ts";
|
|
|
|
const SCENARIO = "built-store-cross-node";
|
|
const CONTROL = "anchor";
|
|
const NODE = "node2";
|
|
const ANCHOR = "192.0.2.10";
|
|
const REGISTRY = `${ANCHOR}:5000`;
|
|
const BASE = { module: "mesh-tools", repo: "mesh-tools", path: "" };
|
|
|
|
const capability = await labIsUsable();
|
|
const binary = hostBinaryPath();
|
|
const installer = bootstrapBinaryPath();
|
|
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
|
const catalogDir = process.env["MESH_LAB_CATALOG"] ?? "";
|
|
const source = process.env["MESH_LAB_SOURCE"] ?? "";
|
|
const sourceRef = process.env["MESH_LAB_SOURCE_REF"] ?? "";
|
|
const KEEP = !!process.env["MESH_LAB_KEEP"];
|
|
/**
|
|
* MESH_LAB_WARM=1: restore the post-genesis, both-nodes-enrolled state from a snapshot instead of
|
|
* re-running the installer (minutes -> seconds), refused — not silently rebuilt — when the commits
|
|
* it was built from have moved (src/warm.ts). Fresh stays the default: a run that must MEAN
|
|
* something raises from nothing.
|
|
*/
|
|
const warming = process.env["MESH_LAB_WARM"] === "1";
|
|
|
|
function forgeUrl(repo: string): string {
|
|
const override = process.env[`MESH_LAB_SOURCE_${repo.toUpperCase().replaceAll("-", "_")}`];
|
|
if (override) return override;
|
|
return source.replace(/[^/]+\.git$/, `${repo}.git`);
|
|
}
|
|
function refFor(repo: string): string {
|
|
const override = process.env[`MESH_LAB_BUILD_REF_${repo.toUpperCase().replaceAll("-", "_")}`];
|
|
return override ?? process.env["MESH_LAB_BUILD_REF"] ?? "main";
|
|
}
|
|
const baseManifest = process.env["MESH_LAB_BASE_MANIFEST"] ??
|
|
resolve(catalogDir, "..", "..", BASE.repo, "module.json");
|
|
|
|
const skip =
|
|
!capability.usable ? capability.why :
|
|
!binary ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" :
|
|
!installer ? "MESH_LAB_BOOTSTRAP_BINARY is not set to a built mesh-bootstrap" :
|
|
!source ? "MESH_LAB_SOURCE is not set to the repository the control plane is built from" :
|
|
!sourceRef ? "MESH_LAB_SOURCE_REF is not set to the commit to build" :
|
|
!bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation template" :
|
|
!catalogDir || !existsSync(catalogDir) ? "MESH_LAB_CATALOG is not set to mesh-catalog/modules" :
|
|
false;
|
|
|
|
let instanceId = "";
|
|
let raised: GenesisResult;
|
|
|
|
function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; }
|
|
|
|
async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
|
|
const { stdout } = await exec(instanceId, machine, ["sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`], timeoutMs);
|
|
const marker = stdout.lastIndexOf("__exit=");
|
|
if (marker < 0) return { out: stdout, ok: false };
|
|
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
|
|
}
|
|
async function must(machine: string, command: string, timeoutMs?: number): Promise<string> {
|
|
const { out, ok } = await on(machine, command, timeoutMs);
|
|
if (!ok) throw new Error(`${machine}: ${command}\n${out}`);
|
|
return out;
|
|
}
|
|
/** The control plane — retried through the brief recreate window a spec change causes. */
|
|
async function mesh(command: string, timeoutMs?: number): Promise<string> {
|
|
const deadline = Date.now() + (timeoutMs ?? 120_000);
|
|
for (;;) {
|
|
const got = await on(CONTROL, `docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
|
|
if (got.ok) return got.out;
|
|
if (!/is not running|No such container/.test(got.out) || Date.now() > deadline) {
|
|
throw new Error(`${CONTROL}: mesh ${command}\n${got.out}`);
|
|
}
|
|
await new Promise((r) => setTimeout(r, 5_000));
|
|
}
|
|
}
|
|
function tokenFrom(said: string): string {
|
|
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
|
|
assert.ok(found, `no token in:\n${said}`);
|
|
return found;
|
|
}
|
|
|
|
/** Register a committed manifest VERBATIM — the point of this bed: no rewriting, ever. */
|
|
async function registerModule(module: string, manifest: string): Promise<string> {
|
|
assert.ok(existsSync(manifest), `no manifest for ${module} at ${manifest}`);
|
|
const onMachine = `/tmp/${module}.json`;
|
|
await push(instanceId, CONTROL, manifest, onMachine);
|
|
await must(CONTROL, `docker cp ${onMachine} mesh-controller:/${module}.json`);
|
|
return mesh(`module add /${module}.json`);
|
|
}
|
|
|
|
/** Build a module with the mesh's own builder, issue its account, and assign it to a node. */
|
|
async function buildAndAssign(module: string, node: string, opts?: { build?: boolean }): Promise<void> {
|
|
await registerModule(module, resolve(catalogDir, module, "module.json"));
|
|
if (opts?.build !== false) {
|
|
const built = await mesh(
|
|
`build ${forgeUrl("mesh-catalog")} --path modules/${module} --ref ${refFor("mesh-catalog")} --wait 1200s`,
|
|
1_500_000);
|
|
assert.doesNotMatch(built, /failed/i, built);
|
|
}
|
|
const manifest = readFileSync(resolve(catalogDir, module, "module.json"), "utf8");
|
|
if (manifest.includes("MESH_BROKER_FILE")) {
|
|
const issued = await mesh(`module issue ${module} --node ${node}`);
|
|
assert.match(issued, /scoped to what it (emits and consumes|consumes and emits)/,
|
|
`the broker account for ${module} was not issued:\n${issued}`);
|
|
}
|
|
await mesh(`assign ${node} ${module}`);
|
|
await mesh(`push ${node}`, 600_000);
|
|
}
|
|
|
|
async function waitForContainer(node: string, container: string, seconds = 300): Promise<string> {
|
|
const deadline = Date.now() + seconds * 1_000;
|
|
let last = "";
|
|
while (Date.now() < deadline) {
|
|
const ps = (await on(node, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out;
|
|
last = ps;
|
|
const line = ps.split("\n").find((l) => l.split("\t")[0]?.trim() === container);
|
|
if (line && /^Up /.test(line.split("\t")[1]?.trim() ?? "")) return ps;
|
|
await new Promise((r) => setTimeout(r, 5_000));
|
|
}
|
|
const logs = (await on(node, `docker logs --tail 20 ${container} 2>&1`)).out;
|
|
throw new Error(`${container} never came up on ${node}:\n${last}\n--- logs ---\n${logs}`);
|
|
}
|
|
|
|
before(async () => {
|
|
if (skip) return;
|
|
if (warming) {
|
|
const said = await ready(SCENARIO);
|
|
if (said.use === "restore") {
|
|
instanceId = said.instanceId;
|
|
const seconds = await returnTo(instanceId);
|
|
// A snapshot captures disk, not memory: the restore reboots. anchor's host is a systemd
|
|
// service (genesis hostService) and comes back on its own; node2's was hand-started.
|
|
await must(NODE, `pgrep -x mesh-host >/dev/null || (nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3)`);
|
|
const back = (await on(CONTROL, `docker ps --format '{{.Names}}'`)).out;
|
|
assert.match(back, /mesh-controller/, `the control plane did not come back after restore:\n${back}`);
|
|
console.log(`warm: returned ${instanceId} to its post-genesis state in ${seconds.toFixed(1)}s`);
|
|
return;
|
|
}
|
|
console.log(`warm: raising fresh — ${said.why}`);
|
|
}
|
|
const bed = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
|
|
onProgress: (m) => console.log(`raise: ${m}`),
|
|
});
|
|
instanceId = bed.instanceId;
|
|
console.log(`INSTANCE ${instanceId}${KEEP ? " (KEEP)" : ""}`);
|
|
console.log("NOTHING WAS LOADED: this scenario names no images; the mesh builds or pulls everything.");
|
|
|
|
// Genesis: the installer raises anchor into a mesh of one, BUILDING the control plane, and
|
|
// leaves the anchor enrolled with an agent running (hostService).
|
|
raised = await genesis({
|
|
instanceId,
|
|
node: CONTROL,
|
|
installer: installer as string,
|
|
catalogDir,
|
|
bundleTemplate: foundationBundle(bundle, []),
|
|
registry: REGISTRY,
|
|
source,
|
|
sourceRef,
|
|
toolsSource: forgeUrl(BASE.repo),
|
|
toolsRef: refFor(BASE.repo),
|
|
catalogSource: forgeUrl("mesh-catalog"),
|
|
catalogRef: refFor("mesh-catalog"),
|
|
sdkSource: forgeUrl("mesh-sdk"),
|
|
sdkRef: refFor("mesh-sdk"),
|
|
site: "hosting",
|
|
hostService: true,
|
|
...(binary ? { hostBinary: binary } : {}),
|
|
log: (m) => console.log(m),
|
|
});
|
|
if (!raised.ok) throw new Error(`${raised.step || "genesis"}: ${raised.why}\n\n${raised.report.join("\n")}`);
|
|
|
|
// node2 joins the mesh: a token against the anchor's public broker endpoint, then an agent.
|
|
await mesh(`node add ${NODE}`);
|
|
const token = tokenFrom(await mesh(`token issue --node ${NODE}`));
|
|
const said = await must(NODE, `${HOST_PATH} enrol --token ${quote(token)}`);
|
|
assert.match(said, new RegExp(`enrolled as ${NODE}`), said);
|
|
await must(NODE, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
|
|
|
|
// The expensive, deterministic part is done: genesis ran and both machines are enrolled. Keep it.
|
|
if (warming) {
|
|
const warm = await keep(SCENARIO, instanceId);
|
|
console.log(`warm: ${warm.instanceId} kept, against ` +
|
|
Object.entries(warm.against).map(([n, c]) => `${n} ${c}`).join(", "));
|
|
}
|
|
}, { timeout: 7_200_000 });
|
|
|
|
after(async () => {
|
|
if (warming) { console.log(`warm — ${instanceId} stays; \`mesh-lab warm cool\` retires it`); return; }
|
|
if (KEEP) { console.log(`MESH_LAB_KEEP set — leaving ${instanceId} standing`); return; }
|
|
if (instanceId) await destroy(instanceId);
|
|
await destroyAll(`${SCENARIO}-`);
|
|
}, { timeout: 900_000 });
|
|
|
|
test("a joined node's consumers open the store and broker the mesh built and adopted, over the overlay", {
|
|
skip, timeout: 3_600_000,
|
|
}, async () => {
|
|
// The overlay, so bindings carry `.internal` names; the firewall, so from:mesh is what admits them.
|
|
await mesh(`overlay place ${CONTROL} --hub --endpoint ${ANCHOR}:51820 --site hosting`);
|
|
await mesh(`overlay place ${NODE} --site hosting`);
|
|
await mesh(`assign ${CONTROL} networking`);
|
|
await mesh(`assign ${NODE} networking`);
|
|
|
|
// The networking module carries the registry trust (ADR 0082): a merged daemon.json naming the
|
|
// store's internal name, and a docker restart when it first lands. It must be ON both machines
|
|
// before anything builds or pulls — the builder pushes to anchor.internal:5000 the moment the
|
|
// first build finishes. Pushed and WAITED for, because the restart bounces the runtime and an
|
|
// apply in flight retries.
|
|
for (const machine of [CONTROL, NODE]) {
|
|
console.log(await mesh(`push ${machine}`, 600_000));
|
|
const deadline = Date.now() + 300_000;
|
|
let trusted = false;
|
|
while (Date.now() < deadline) {
|
|
const got = await on(machine, `grep -s "anchor.internal:5000" /etc/docker/daemon.json && docker info --format '{{json .RegistryConfig.IndexConfigs}}' 2>/dev/null | grep -q "anchor.internal:5000" && echo TRUSTED`);
|
|
if (/TRUSTED/.test(got.out)) { trusted = true; break; }
|
|
await new Promise((r) => setTimeout(r, 5_000));
|
|
}
|
|
// A failure here has two distinguishable shapes, so the dump carries both: a declaration that
|
|
// never named the trust (the controller composed without it — issues 042/048 as a race), and
|
|
// one that named it and was never applied (delivery or apply). mesh-host's log says which.
|
|
assert.ok(trusted, `${machine}'s runtime never learned the registry trust:\n` +
|
|
(await on(machine, `cat /etc/docker/daemon.json 2>&1; docker info 2>&1 | tail -20`)).out +
|
|
`\n--- ${machine} mesh-host.log ---\n` +
|
|
(await on(machine, `tail -60 /var/log/mesh-host.log 2>&1`)).out +
|
|
`\n--- ${machine} declared registry-trust? ---\n` +
|
|
(await on(machine, `base64 -d < /var/lib/mesh-host/declared.json 2>/dev/null | grep -c registry-trust; python3 -c "import json,base64; d=json.load(open('/var/lib/mesh-host/declared.json')); print('registry-trust' in base64.b64decode(d['declaration']).decode())" 2>&1`)).out);
|
|
}
|
|
|
|
// The shared base first — every module with code of its own stands on it.
|
|
await registerModule(BASE.module, baseManifest);
|
|
const base = await mesh(`build ${forgeUrl(BASE.repo)} --ref ${refFor(BASE.repo)} --wait 1200s`, 1_500_000);
|
|
assert.doesNotMatch(base, /failed/i, base);
|
|
|
|
// Genesis already adopted the store as the postgres module (superuser accepted), and installed
|
|
// nftables and the catalogue — verified rather than redone. The broker is the one foundation
|
|
// half genesis leaves to the mesh proper: adopt it here, BUILT by the mesh's own builder.
|
|
await waitForContainer(CONTROL, "mesh-postgres", 120);
|
|
await buildAndAssign("lavinmq", CONTROL);
|
|
await waitForContainer(CONTROL, "mesh-lavinmq", 600);
|
|
|
|
// The consumer on the joined node — built by the mesh, delivered from its registry. One consumer,
|
|
// amqp-ping, because it is one of the eight modules the mesh can actually build; the catalogue's
|
|
// DB consumers all lack a build section (hq issue 060), so the store's cross-node proof stays with
|
|
// the stocked-image bed until one of them gains one. NOTE: this delivery is the path hq issues
|
|
// 042 (a node has no registry account) and 048 (nothing makes a machine trust the registry) leave
|
|
// open — this bed is their honest gate, red until they are fixed.
|
|
await buildAndAssign("amqp-ping", NODE);
|
|
|
|
// 057: the provider node is composed again so its provisioners learn of the remote consumers.
|
|
await mesh(`push ${CONTROL}`, 600_000);
|
|
|
|
// THE BROKER, cross-node: amqp-ping's binding names the control-node's overlay name, its vhost is
|
|
// minted on the one broker, and it holds the connection.
|
|
try {
|
|
await waitForContainer(NODE, "amqp-ping", 600);
|
|
} catch (err) {
|
|
const hostLog = (await on(NODE, `tail -40 /var/log/mesh-host.log`)).out;
|
|
throw new Error(`${(err as Error).message}\n--- ${NODE} mesh-host.log (a pull that failed ` +
|
|
`never reaches container logs; hq issues 042/048) ---\n${hostLog}`);
|
|
}
|
|
const amqpBound = (await on(NODE, `cat /var/lib/amqp-ping/amqp.json 2>&1`)).out;
|
|
assert.match(amqpBound, new RegExp(`${CONTROL}\\.internal`),
|
|
`the amqp grant does not point at the control-node over the overlay:\n${amqpBound}`);
|
|
{
|
|
const deadline = Date.now() + 240_000;
|
|
let vhosts = "";
|
|
while (Date.now() < deadline) {
|
|
vhosts = (await on(CONTROL, `docker exec mesh-broker lavinmqctl list_vhosts 2>&1`)).out;
|
|
if (new RegExp(`${NODE}|amqp-ping`).test(vhosts)) break;
|
|
await new Promise((r) => setTimeout(r, 5_000));
|
|
}
|
|
assert.match(vhosts, new RegExp(`${NODE}|amqp-ping`),
|
|
`no vhost was minted on the one broker for the joined consumer:\n${vhosts}\n` +
|
|
`--- provisioner ---\n${(await on(CONTROL, `docker logs mesh-lavinmq 2>&1 | tail -20`)).out}\n` +
|
|
`--- consumer ---\n${(await on(NODE, `docker logs amqp-ping 2>&1 | tail -20`)).out}`);
|
|
}
|
|
|
|
// Adoption did not cost the foundation: the containers genesis raised are still the ones running.
|
|
const up = await must(CONTROL, `docker ps --format '{{.Names}}'`);
|
|
for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) {
|
|
assert.match(up, new RegExp(`(^|\\n)${c}(\\n|$)`), `${c} did not survive adoption:\n${up}`);
|
|
}
|
|
// Steady a moment, then confirm the consumer did not crash-loop after connecting.
|
|
await new Promise((r) => setTimeout(r, 15_000));
|
|
assert.match((await on(NODE, `docker ps --format '{{.Names}}\t{{.Status}}'`)).out,
|
|
/amqp-ping\tUp/, `amqp-ping did not stay up on ${NODE}`);
|
|
|
|
console.log(`amqp: ${amqpBound.trim().slice(0, 160)}`);
|
|
});
|