jschoubben
064a7a0934
- The RestartCount==0 assertion cannot discriminate the 058 fix: the consumer is built after its broker is already up, so patient and exit-on-unreachable code both connect first-try; and restart-on recreates reset the count. Downgraded to an honest liveness check and the comment now points at the mesh-tools unit test as the real proof. - The trust-failure dump ran base64 -d over declared.json, which is JSON (not base64), so it always reported 'no trust' — removed; the adjacent python check that decodes the inner declaration field is kept. - The 063 comment claimed the vhost is re-listed after the restart; the code only runs a TCP probe. Comment corrected to what the code proves, and the docker-proxy-vs-DNAT coupling is noted.
368 lines
20 KiB
TypeScript
368 lines
20 KiB
TypeScript
/**
|
|
* THE NO-FAKE MULTI-NODE GATE: two machines, everything built by the mesh itself.
|
|
*
|
|
* The rewrite-based multi-node beds pre-stock runtime images and rewrite each manifest's
|
|
* placeholder digest to whatever they stocked — bed code doing the builder's job, which means the
|
|
* builder's job was never under test. This bed removes the shortcut. The scenario names NO images;
|
|
* the committed manifests are registered VERBATIM; the only thing that ever turns
|
|
* `mesh-runtime-<m>@sha256:0000…` (or an `artifact:` reference) into a real digest is the mesh's
|
|
* own builder, exactly as in production.
|
|
*
|
|
* What it proves, end to end:
|
|
* 1. The installer raises `anchor` into a mesh of one — building the control plane (ADR 0073),
|
|
* standing up the registry and the builder.
|
|
* 2. The mesh BUILDS the shared base, then postgres and lavinmq, from the forge — and adopts the
|
|
* foundation's own store and broker as those modules (ADR 0078), the store's genesis superuser
|
|
* carried in through `secret accept` (the same act as hq phase3 deliverSuperuser).
|
|
* 3. `node2` joins, and its consumers are BUILT and delivered the same way: `amqp-ping` opens the
|
|
* one broker and `letta` gets a database on the one store — both across the overlay, admitted
|
|
* by the firewall the nftables module derives (issues 055/056/057 in one bed).
|
|
*
|
|
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BOOTSTRAP_BINARY=.../mesh-bootstrap
|
|
* MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock (the TEMPLATE)
|
|
* MESH_LAB_CATALOG=.../mesh-catalog/modules
|
|
* MESH_LAB_SOURCE=git://<forge>/mesh-controller.git MESH_LAB_SOURCE_REF=<commit>
|
|
* MESH_LAB_BUILD_REF=<branch or commit for module builds, default main>
|
|
*/
|
|
import { test, before, after } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { existsSync, readFileSync } from "node:fs";
|
|
import { resolve } from "node:path";
|
|
import { loadScenario } from "../../src/declaration/parse.ts";
|
|
import { raise } from "../../src/lifecycle/raise.ts";
|
|
import { destroy, exec, push } from "../../src/lifecycle/operate.ts";
|
|
import { bootstrapBinaryPath, hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
|
import { labIsUsable, destroyAll, foundationBundle } from "./harness.ts";
|
|
import { genesis, type GenesisResult } from "./genesis.ts";
|
|
import { keep, ready, returnTo } from "../../src/warm.ts";
|
|
|
|
const SCENARIO = "built-store-cross-node";
|
|
const CONTROL = "anchor";
|
|
const NODE = "node2";
|
|
const ANCHOR = "192.0.2.10";
|
|
const REGISTRY = `${ANCHOR}:5000`;
|
|
const BASE = { module: "mesh-tools", repo: "mesh-tools", path: "" };
|
|
|
|
const capability = await labIsUsable();
|
|
const binary = hostBinaryPath();
|
|
const installer = bootstrapBinaryPath();
|
|
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
|
const catalogDir = process.env["MESH_LAB_CATALOG"] ?? "";
|
|
const source = process.env["MESH_LAB_SOURCE"] ?? "";
|
|
const sourceRef = process.env["MESH_LAB_SOURCE_REF"] ?? "";
|
|
const KEEP = !!process.env["MESH_LAB_KEEP"];
|
|
/**
|
|
* MESH_LAB_WARM=1: restore the post-genesis, both-nodes-enrolled state from a snapshot instead of
|
|
* re-running the installer (minutes -> seconds), refused — not silently rebuilt — when the commits
|
|
* it was built from have moved (src/warm.ts). Fresh stays the default: a run that must MEAN
|
|
* something raises from nothing.
|
|
*/
|
|
const warming = process.env["MESH_LAB_WARM"] === "1";
|
|
|
|
function forgeUrl(repo: string): string {
|
|
const override = process.env[`MESH_LAB_SOURCE_${repo.toUpperCase().replaceAll("-", "_")}`];
|
|
if (override) return override;
|
|
return source.replace(/[^/]+\.git$/, `${repo}.git`);
|
|
}
|
|
function refFor(repo: string): string {
|
|
const override = process.env[`MESH_LAB_BUILD_REF_${repo.toUpperCase().replaceAll("-", "_")}`];
|
|
return override ?? process.env["MESH_LAB_BUILD_REF"] ?? "main";
|
|
}
|
|
const baseManifest = process.env["MESH_LAB_BASE_MANIFEST"] ??
|
|
resolve(catalogDir, "..", "..", BASE.repo, "module.json");
|
|
|
|
const skip =
|
|
!capability.usable ? capability.why :
|
|
!binary ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" :
|
|
!installer ? "MESH_LAB_BOOTSTRAP_BINARY is not set to a built mesh-bootstrap" :
|
|
!source ? "MESH_LAB_SOURCE is not set to the repository the control plane is built from" :
|
|
!sourceRef ? "MESH_LAB_SOURCE_REF is not set to the commit to build" :
|
|
!bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation template" :
|
|
!catalogDir || !existsSync(catalogDir) ? "MESH_LAB_CATALOG is not set to mesh-catalog/modules" :
|
|
false;
|
|
|
|
let instanceId = "";
|
|
let raised: GenesisResult;
|
|
|
|
function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; }
|
|
|
|
async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
|
|
const { stdout } = await exec(instanceId, machine, ["sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`], timeoutMs);
|
|
const marker = stdout.lastIndexOf("__exit=");
|
|
if (marker < 0) return { out: stdout, ok: false };
|
|
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
|
|
}
|
|
async function must(machine: string, command: string, timeoutMs?: number): Promise<string> {
|
|
const { out, ok } = await on(machine, command, timeoutMs);
|
|
if (!ok) throw new Error(`${machine}: ${command}\n${out}`);
|
|
return out;
|
|
}
|
|
/** The control plane — retried through the brief recreate window a spec change causes. */
|
|
async function mesh(command: string, timeoutMs?: number): Promise<string> {
|
|
const deadline = Date.now() + (timeoutMs ?? 120_000);
|
|
for (;;) {
|
|
const got = await on(CONTROL, `docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
|
|
if (got.ok) return got.out;
|
|
if (!/is not running|No such container/.test(got.out) || Date.now() > deadline) {
|
|
throw new Error(`${CONTROL}: mesh ${command}\n${got.out}`);
|
|
}
|
|
await new Promise((r) => setTimeout(r, 5_000));
|
|
}
|
|
}
|
|
function tokenFrom(said: string): string {
|
|
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
|
|
assert.ok(found, `no token in:\n${said}`);
|
|
return found;
|
|
}
|
|
|
|
/** Register a committed manifest VERBATIM — the point of this bed: no rewriting, ever. */
|
|
async function registerModule(module: string, manifest: string): Promise<string> {
|
|
assert.ok(existsSync(manifest), `no manifest for ${module} at ${manifest}`);
|
|
const onMachine = `/tmp/${module}.json`;
|
|
await push(instanceId, CONTROL, manifest, onMachine);
|
|
await must(CONTROL, `docker cp ${onMachine} mesh-controller:/${module}.json`);
|
|
return mesh(`module add /${module}.json`);
|
|
}
|
|
|
|
/** Build a module with the mesh's own builder, issue its account, and assign it to a node. */
|
|
async function buildAndAssign(module: string, node: string, opts?: { build?: boolean }): Promise<void> {
|
|
await registerModule(module, resolve(catalogDir, module, "module.json"));
|
|
if (opts?.build !== false) {
|
|
const built = await mesh(
|
|
`build ${forgeUrl("mesh-catalog")} --path modules/${module} --ref ${refFor("mesh-catalog")} --wait 1200s`,
|
|
1_500_000);
|
|
assert.doesNotMatch(built, /failed/i, built);
|
|
}
|
|
const manifest = readFileSync(resolve(catalogDir, module, "module.json"), "utf8");
|
|
if (manifest.includes("MESH_BROKER_FILE")) {
|
|
const issued = await mesh(`module issue ${module} --node ${node}`);
|
|
assert.match(issued, /scoped to what it (emits and consumes|consumes and emits)/,
|
|
`the broker account for ${module} was not issued:\n${issued}`);
|
|
}
|
|
await mesh(`assign ${node} ${module}`);
|
|
await mesh(`push ${node}`, 600_000);
|
|
}
|
|
|
|
async function waitForContainer(node: string, container: string, seconds = 300): Promise<string> {
|
|
const deadline = Date.now() + seconds * 1_000;
|
|
let last = "";
|
|
while (Date.now() < deadline) {
|
|
const ps = (await on(node, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out;
|
|
last = ps;
|
|
const line = ps.split("\n").find((l) => l.split("\t")[0]?.trim() === container);
|
|
if (line && /^Up /.test(line.split("\t")[1]?.trim() ?? "")) return ps;
|
|
await new Promise((r) => setTimeout(r, 5_000));
|
|
}
|
|
const logs = (await on(node, `docker logs --tail 20 ${container} 2>&1`)).out;
|
|
throw new Error(`${container} never came up on ${node}:\n${last}\n--- logs ---\n${logs}`);
|
|
}
|
|
|
|
before(async () => {
|
|
if (skip) return;
|
|
if (warming) {
|
|
const said = await ready(SCENARIO);
|
|
if (said.use === "restore") {
|
|
instanceId = said.instanceId;
|
|
const seconds = await returnTo(instanceId);
|
|
// A snapshot captures disk, not memory: the restore reboots. anchor's host is a systemd
|
|
// service (genesis hostService) and comes back on its own; node2's was hand-started.
|
|
await must(NODE, `pgrep -x mesh-host >/dev/null || (nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3)`);
|
|
const back = (await on(CONTROL, `docker ps --format '{{.Names}}'`)).out;
|
|
assert.match(back, /mesh-controller/, `the control plane did not come back after restore:\n${back}`);
|
|
console.log(`warm: returned ${instanceId} to its post-genesis state in ${seconds.toFixed(1)}s`);
|
|
return;
|
|
}
|
|
console.log(`warm: raising fresh — ${said.why}`);
|
|
}
|
|
const bed = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
|
|
onProgress: (m) => console.log(`raise: ${m}`),
|
|
});
|
|
instanceId = bed.instanceId;
|
|
console.log(`INSTANCE ${instanceId}${KEEP ? " (KEEP)" : ""}`);
|
|
console.log("NOTHING WAS LOADED: this scenario names no images; the mesh builds or pulls everything.");
|
|
|
|
// Genesis: the installer raises anchor into a mesh of one, BUILDING the control plane, and
|
|
// leaves the anchor enrolled with an agent running (hostService).
|
|
raised = await genesis({
|
|
instanceId,
|
|
node: CONTROL,
|
|
installer: installer as string,
|
|
catalogDir,
|
|
bundleTemplate: foundationBundle(bundle, []),
|
|
registry: REGISTRY,
|
|
source,
|
|
sourceRef,
|
|
toolsSource: forgeUrl(BASE.repo),
|
|
toolsRef: refFor(BASE.repo),
|
|
catalogSource: forgeUrl("mesh-catalog"),
|
|
catalogRef: refFor("mesh-catalog"),
|
|
sdkSource: forgeUrl("mesh-sdk"),
|
|
sdkRef: refFor("mesh-sdk"),
|
|
site: "hosting",
|
|
hostService: true,
|
|
...(binary ? { hostBinary: binary } : {}),
|
|
log: (m) => console.log(m),
|
|
});
|
|
if (!raised.ok) throw new Error(`${raised.step || "genesis"}: ${raised.why}\n\n${raised.report.join("\n")}`);
|
|
|
|
// node2 joins the mesh: a token against the anchor's public broker endpoint, then an agent.
|
|
await mesh(`node add ${NODE}`);
|
|
const token = tokenFrom(await mesh(`token issue --node ${NODE}`));
|
|
const said = await must(NODE, `${HOST_PATH} enrol --token ${quote(token)}`);
|
|
assert.match(said, new RegExp(`enrolled as ${NODE}`), said);
|
|
await must(NODE, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
|
|
|
|
// The expensive, deterministic part is done: genesis ran and both machines are enrolled. Keep it.
|
|
if (warming) {
|
|
const warm = await keep(SCENARIO, instanceId);
|
|
console.log(`warm: ${warm.instanceId} kept, against ` +
|
|
Object.entries(warm.against).map(([n, c]) => `${n} ${c}`).join(", "));
|
|
}
|
|
}, { timeout: 7_200_000 });
|
|
|
|
after(async () => {
|
|
if (warming) { console.log(`warm — ${instanceId} stays; \`mesh-lab warm cool\` retires it`); return; }
|
|
if (KEEP) { console.log(`MESH_LAB_KEEP set — leaving ${instanceId} standing`); return; }
|
|
if (instanceId) await destroy(instanceId);
|
|
await destroyAll(`${SCENARIO}-`);
|
|
}, { timeout: 900_000 });
|
|
|
|
test("a joined node's consumers open the store and broker the mesh built and adopted, over the overlay", {
|
|
skip, timeout: 3_600_000,
|
|
}, async () => {
|
|
// The overlay, so bindings carry `.internal` names; the firewall, so from:mesh is what admits them.
|
|
await mesh(`overlay place ${CONTROL} --hub --endpoint ${ANCHOR}:51820 --site hosting`);
|
|
await mesh(`overlay place ${NODE} --site hosting`);
|
|
await mesh(`assign ${CONTROL} networking`);
|
|
await mesh(`assign ${NODE} networking`);
|
|
|
|
// The networking module carries the registry trust (ADR 0082): a merged daemon.json naming the
|
|
// store's internal name, and a docker restart when it first lands. It must be ON both machines
|
|
// before anything builds or pulls — the builder pushes to anchor.internal:5000 the moment the
|
|
// first build finishes. Pushed and WAITED for, because the restart bounces the runtime and an
|
|
// apply in flight retries.
|
|
for (const machine of [CONTROL, NODE]) {
|
|
console.log(await mesh(`push ${machine}`, 600_000));
|
|
const deadline = Date.now() + 300_000;
|
|
let trusted = false;
|
|
while (Date.now() < deadline) {
|
|
const got = await on(machine, `grep -s "anchor.internal:5000" /etc/docker/daemon.json && docker info --format '{{json .RegistryConfig.IndexConfigs}}' 2>/dev/null | grep -q "anchor.internal:5000" && echo TRUSTED`);
|
|
if (/TRUSTED/.test(got.out)) { trusted = true; break; }
|
|
await new Promise((r) => setTimeout(r, 5_000));
|
|
}
|
|
// A failure here has two distinguishable shapes, so the dump carries both: a declaration that
|
|
// never named the trust (the controller composed without it — issues 042/048 as a race), and
|
|
// one that named it and was never applied (delivery or apply). mesh-host's log says which.
|
|
assert.ok(trusted, `${machine}'s runtime never learned the registry trust:\n` +
|
|
(await on(machine, `cat /etc/docker/daemon.json 2>&1; docker info 2>&1 | tail -20`)).out +
|
|
`\n--- ${machine} mesh-host.log ---\n` +
|
|
(await on(machine, `tail -60 /var/log/mesh-host.log 2>&1`)).out +
|
|
`\n--- ${machine} declared registry-trust? ---\n` +
|
|
// declared.json is {"declaration":"<base64>","signature":"<base64>"} — JSON, not base64
|
|
// itself, so only the inner `declaration` field is decoded. (An earlier `base64 -d` on the
|
|
// whole file always errored and reported "no trust" whether or not the trust was declared.)
|
|
(await on(machine, `python3 -c "import json,base64; d=json.load(open('/var/lib/mesh-host/declared.json')); print('registry-trust' in base64.b64decode(d['declaration']).decode())" 2>&1`)).out);
|
|
}
|
|
|
|
// The shared base first — every module with code of its own stands on it.
|
|
await registerModule(BASE.module, baseManifest);
|
|
const base = await mesh(`build ${forgeUrl(BASE.repo)} --ref ${refFor(BASE.repo)} --wait 1200s`, 1_500_000);
|
|
assert.doesNotMatch(base, /failed/i, base);
|
|
|
|
// Genesis already adopted the store as the postgres module (superuser accepted), and installed
|
|
// nftables and the catalogue — verified rather than redone. The broker is the one foundation
|
|
// half genesis leaves to the mesh proper: adopt it here, BUILT by the mesh's own builder.
|
|
await waitForContainer(CONTROL, "mesh-postgres", 120);
|
|
await buildAndAssign("lavinmq", CONTROL);
|
|
await waitForContainer(CONTROL, "mesh-lavinmq", 600);
|
|
|
|
// The consumer on the joined node — built by the mesh, delivered from its registry. One consumer,
|
|
// amqp-ping, because it is one of the eight modules the mesh can actually build; the catalogue's
|
|
// DB consumers all lack a build section (hq issue 060), so the store's cross-node proof stays with
|
|
// the stocked-image bed until one of them gains one. NOTE: this delivery is the path hq issues
|
|
// 042 (a node has no registry account) and 048 (nothing makes a machine trust the registry) leave
|
|
// open — this bed is their honest gate, red until they are fixed.
|
|
await buildAndAssign("amqp-ping", NODE);
|
|
|
|
// No second push of the provider node. Issue 057's fix makes one push sufficient: pushing the
|
|
// consumer's node cascades to the machines whose declaration changed because of it — the
|
|
// provider learns of the remote consumer from that same act. This bed is the enforcement: put
|
|
// the workaround push back and the cascade is untested again.
|
|
|
|
// THE BROKER, cross-node: amqp-ping's binding names the control-node's overlay name, its vhost is
|
|
// minted on the one broker, and it holds the connection.
|
|
try {
|
|
await waitForContainer(NODE, "amqp-ping", 600);
|
|
} catch (err) {
|
|
const hostLog = (await on(NODE, `tail -40 /var/log/mesh-host.log`)).out;
|
|
throw new Error(`${(err as Error).message}\n--- ${NODE} mesh-host.log (a pull that failed ` +
|
|
`never reaches container logs; hq issues 042/048) ---\n${hostLog}`);
|
|
}
|
|
const amqpBound = (await on(NODE, `cat /var/lib/amqp-ping/amqp.json 2>&1`)).out;
|
|
assert.match(amqpBound, new RegExp(`${CONTROL}\\.internal`),
|
|
`the amqp grant does not point at the control-node over the overlay:\n${amqpBound}`);
|
|
{
|
|
const deadline = Date.now() + 240_000;
|
|
let vhosts = "";
|
|
while (Date.now() < deadline) {
|
|
vhosts = (await on(CONTROL, `docker exec mesh-broker lavinmqctl list_vhosts 2>&1`)).out;
|
|
if (new RegExp(`${NODE}|amqp-ping`).test(vhosts)) break;
|
|
await new Promise((r) => setTimeout(r, 5_000));
|
|
}
|
|
assert.match(vhosts, new RegExp(`${NODE}|amqp-ping`),
|
|
`no vhost was minted on the one broker for the joined consumer:\n${vhosts}\n` +
|
|
`--- provisioner ---\n${(await on(CONTROL, `docker logs mesh-lavinmq 2>&1 | tail -20`)).out}\n` +
|
|
`--- consumer ---\n${(await on(NODE, `docker logs amqp-ping 2>&1 | tail -20`)).out}`);
|
|
}
|
|
|
|
// Adoption did not cost the foundation: the containers genesis raised are still the ones running.
|
|
const up = await must(CONTROL, `docker ps --format '{{.Names}}'`);
|
|
for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) {
|
|
assert.match(up, new RegExp(`(^|\\n)${c}(\\n|$)`), `${c} did not survive adoption:\n${up}`);
|
|
}
|
|
// Steady a moment, then confirm the consumer is up and did not crash-loop after connecting.
|
|
//
|
|
// **This is a liveness check, not the proof of hq issue 058.** By the time amqp-ping is built
|
|
// and assigned, its broker is already up and reachable, so both the patient runtime and the
|
|
// old exit-on-unreachable code connect on the first try — a green here does not discriminate
|
|
// the fix. And RestartCount counts in-place restarts of the *current* container, which a
|
|
// restart-on recreate (amqp-ping declares `restart-on: [amqp-env]`) resets to zero. The honest
|
|
// proof of the patient-reconnect logic is the mesh-tools unit test of fatalBrokerReason; what
|
|
// this asserts is only that the consumer settled and stayed up.
|
|
await new Promise((r) => setTimeout(r, 15_000));
|
|
assert.match((await on(NODE, `docker ps --format '{{.Names}}\t{{.Status}}'`)).out,
|
|
/amqp-ping\tUp/, `amqp-ping did not stay up on ${NODE}:\n` +
|
|
(await on(NODE, `docker logs amqp-ping 2>&1 | tail -20`)).out);
|
|
|
|
// The broker survives a restart as a reachable thing, not just a running one (hq issue 063).
|
|
// The foundation's ports are opened from anywhere on input, but the broker is a published
|
|
// container port — so a cross-node dial is forwarded, and until 063 the forward chain carried
|
|
// no foundation rule: the joined node reached the broker only until its first connection's
|
|
// conntrack entry dropped. Restarting the broker here drops it deliberately, then a FRESH TCP
|
|
// connection from the joined node must complete — a new 5-tuple in state NEW cannot ride the
|
|
// old established entry, so only the forward rule 063 adds can carry it. (This asserts the
|
|
// reachability the fix restores; the vhost minting itself is already asserted above.)
|
|
//
|
|
// The probe reaches the forward chain because this runtime DNATs published ports (no userland
|
|
// proxy) — a cross-node dial is redirected to the container and forwarded. Were docker-proxy
|
|
// ever enabled, the same dial would terminate on the host and be satisfied by the INPUT rule,
|
|
// masking a missing forward rule; the lab's runtime does not use it, so the probe is honest here.
|
|
await must(CONTROL, `docker restart mesh-broker`, 120_000);
|
|
{
|
|
const deadline = Date.now() + 180_000;
|
|
let reachable = "";
|
|
while (Date.now() < deadline) {
|
|
reachable = (await on(NODE,
|
|
`timeout 5 bash -c 'cat < /dev/null > /dev/tcp/${ANCHOR}/5671' 2>&1 && echo REACHED`)).out;
|
|
if (/REACHED/.test(reachable)) break;
|
|
await new Promise((r) => setTimeout(r, 5_000));
|
|
}
|
|
assert.match(reachable, /REACHED/,
|
|
`after the broker restarted, ${NODE} cannot reach it at ${ANCHOR}:5671 — the foundation's ` +
|
|
`forward rule is missing (hq issue 063):\n` +
|
|
(await on(CONTROL, `nft list ruleset 2>/dev/null | sed -n '/chain forward/,/}/p'`)).out);
|
|
}
|
|
|
|
console.log(`amqp: ${amqpBound.trim().slice(0, 160)}`);
|
|
});
|