scenarios/built-store-cross-node.yml

@@ -0,0 +1,54 @@
+# TWO MACHINES, AND NOTHING FAKED. The no-fake multi-node gate.
+#
+# The one-node scenario proves a machine given nothing but a container runtime ends up with a mesh
+# that BUILT everything it runs. This is its two-machine sibling, and the multi-node claim it adds
+# is the one the rewrite-based beds could not honestly make: every image on either machine was
+# pulled from the internet or built by the mesh's own builder — the bed rewrites nothing, pins
+# nothing, stocks nothing. The builder is the only thing that ever turns a manifest's placeholder
+# into a digest, exactly as in production.
+#
+# anchor is raised into a mesh of one by the installer, adopts the foundation store and broker as
+# the postgres and lavinmq modules (built by the mesh), and node2 joins and runs the consumers —
+# amqp-ping against the one broker, letta against the one store — over the overlay.
+#
+# **There is no `images:` key, and that is the whole point of this file.**
+#
+# EGRESS IS NOT OPTIONAL: with nothing loaded, a sealed machine stops at the installer's first pull.
+#
+#   MESH_LAB_HOST_BINARY=.../mesh-host   MESH_LAB_BOOTSTRAP_BINARY=.../mesh-bootstrap
+#   MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
+#   MESH_LAB_CATALOG=.../mesh-catalog/modules
+#   MESH_LAB_SOURCE=<forge url>   MESH_LAB_SOURCE_REF=<commit>
+scenario: built-store-cross-node
+
+segments:
+  hosting:
+    kind: public
+    cidr: [192.0.2.0/24]
+
+machines:
+  # The address matters: the foundation template names the broker at 192.0.2.10:5671, and a token
+  # carries that verbatim as the endpoint an enrolling node dials.
+  #
+  # Sized like the one-node anchor: store, broker, registry, two control planes during the pivot,
+  # the builder with a Node toolchain and npm cache, and the built module images on top.
+  anchor:
+    at: { segment: hosting, address: [192.0.2.10] }
+    egress: true
+    inbound: allow
+    memory: 12GiB
+    cpus: 6
+    disk: 60GiB
+  # The joined consumer node. It builds nothing — it pulls what the mesh's registry serves and what
+  # its modules name upstream (letta's app image comes from the internet, ~2GB), so it needs egress
+  # and room for images, not build horsepower.
+  node2:
+    at: { segment: hosting, address: [192.0.2.20] }
+    egress: true
+    inbound: allow
+    memory: 6GiB
+    cpus: 4
+    disk: 40GiB
+
+place:
+  all: [host, runtime]

test/integration/built-store-cross-node.test.ts

@@ -0,0 +1,287 @@
+/**
+ * THE NO-FAKE MULTI-NODE GATE: two machines, everything built by the mesh itself.
+ *
+ * The rewrite-based multi-node beds pre-stock runtime images and rewrite each manifest's
+ * placeholder digest to whatever they stocked — bed code doing the builder's job, which means the
+ * builder's job was never under test. This bed removes the shortcut. The scenario names NO images;
+ * the committed manifests are registered VERBATIM; the only thing that ever turns
+ * `mesh-runtime-<m>@sha256:0000…` (or an `artifact:` reference) into a real digest is the mesh's
+ * own builder, exactly as in production.
+ *
+ * What it proves, end to end:
+ *   1. The installer raises `anchor` into a mesh of one — building the control plane (ADR 0073),
+ *      standing up the registry and the builder.
+ *   2. The mesh BUILDS the shared base, then postgres and lavinmq, from the forge — and adopts the
+ *      foundation's own store and broker as those modules (ADR 0078), the store's genesis superuser
+ *      carried in through `secret accept` (the same act as hq phase3 deliverSuperuser).
+ *   3. `node2` joins, and its consumers are BUILT and delivered the same way: `amqp-ping` opens the
+ *      one broker and `letta` gets a database on the one store — both across the overlay, admitted
+ *      by the firewall the nftables module derives (issues 055/056/057 in one bed).
+ *
+ *   MESH_LAB_HOST_BINARY=.../mesh-host   MESH_LAB_BOOTSTRAP_BINARY=.../mesh-bootstrap
+ *   MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock  (the TEMPLATE)
+ *   MESH_LAB_CATALOG=.../mesh-catalog/modules
+ *   MESH_LAB_SOURCE=git://<forge>/mesh-controller.git   MESH_LAB_SOURCE_REF=<commit>
+ *   MESH_LAB_BUILD_REF=<branch or commit for module builds, default main>
+ */
+import { test, before, after } from "node:test";
+import assert from "node:assert/strict";
+import { existsSync } from "node:fs";
+import { resolve } from "node:path";
+import { loadScenario } from "../../src/declaration/parse.ts";
+import { raise } from "../../src/lifecycle/raise.ts";
+import { destroy, exec, push } from "../../src/lifecycle/operate.ts";
+import { bootstrapBinaryPath, hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
+import { labIsUsable, destroyAll, foundationBundle } from "./harness.ts";
+import { genesis, type GenesisResult } from "./genesis.ts";
+
+const SCENARIO = "built-store-cross-node";
+const CONTROL = "anchor";
+const NODE = "node2";
+const ANCHOR = "192.0.2.10";
+const REGISTRY = `${ANCHOR}:5000`;
+const BASE = { module: "mesh-tools", repo: "mesh-tools", path: "" };
+
+const capability = await labIsUsable();
+const binary = hostBinaryPath();
+const installer = bootstrapBinaryPath();
+const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
+const catalogDir = process.env["MESH_LAB_CATALOG"] ?? "";
+const source = process.env["MESH_LAB_SOURCE"] ?? "";
+const sourceRef = process.env["MESH_LAB_SOURCE_REF"] ?? "";
+const KEEP = !!process.env["MESH_LAB_KEEP"];
+
+function forgeUrl(repo: string): string {
+  const override = process.env[`MESH_LAB_SOURCE_${repo.toUpperCase().replaceAll("-", "_")}`];
+  if (override) return override;
+  return source.replace(/[^/]+\.git$/, `${repo}.git`);
+}
+function refFor(repo: string): string {
+  const override = process.env[`MESH_LAB_BUILD_REF_${repo.toUpperCase().replaceAll("-", "_")}`];
+  return override ?? process.env["MESH_LAB_BUILD_REF"] ?? "main";
+}
+const baseManifest = process.env["MESH_LAB_BASE_MANIFEST"] ??
+  resolve(catalogDir, "..", "..", BASE.repo, "module.json");
+
+const skip =
+  !capability.usable ? capability.why :
+  !binary ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" :
+  !installer ? "MESH_LAB_BOOTSTRAP_BINARY is not set to a built mesh-bootstrap" :
+  !source ? "MESH_LAB_SOURCE is not set to the repository the control plane is built from" :
+  !sourceRef ? "MESH_LAB_SOURCE_REF is not set to the commit to build" :
+  !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation template" :
+  !catalogDir || !existsSync(catalogDir) ? "MESH_LAB_CATALOG is not set to mesh-catalog/modules" :
+  false;
+
+let instanceId = "";
+let raised: GenesisResult;
+
+function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; }
+
+async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
+  const { stdout } = await exec(instanceId, machine, ["sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`], timeoutMs);
+  const marker = stdout.lastIndexOf("__exit=");
+  if (marker < 0) return { out: stdout, ok: false };
+  return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
+}
+async function must(machine: string, command: string, timeoutMs?: number): Promise<string> {
+  const { out, ok } = await on(machine, command, timeoutMs);
+  if (!ok) throw new Error(`${machine}: ${command}\n${out}`);
+  return out;
+}
+/** The control plane — retried through the brief recreate window a spec change causes. */
+async function mesh(command: string, timeoutMs?: number): Promise<string> {
+  const deadline = Date.now() + (timeoutMs ?? 120_000);
+  for (;;) {
+    const got = await on(CONTROL, `docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
+    if (got.ok) return got.out;
+    if (!/is not running|No such container/.test(got.out) || Date.now() > deadline) {
+      throw new Error(`${CONTROL}: mesh ${command}\n${got.out}`);
+    }
+    await new Promise((r) => setTimeout(r, 5_000));
+  }
+}
+function tokenFrom(said: string): string {
+  const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
+  assert.ok(found, `no token in:\n${said}`);
+  return found;
+}
+
+/** Register a committed manifest VERBATIM — the point of this bed: no rewriting, ever. */
+async function registerModule(module: string, manifest: string): Promise<string> {
+  assert.ok(existsSync(manifest), `no manifest for ${module} at ${manifest}`);
+  const onMachine = `/tmp/${module}.json`;
+  await push(instanceId, CONTROL, manifest, onMachine);
+  await must(CONTROL, `docker cp ${onMachine} mesh-controller:/${module}.json`);
+  return mesh(`module add /${module}.json`);
+}
+
+/** Build a module with the mesh's own builder, issue its account, and assign it to a node. */
+async function buildAndAssign(module: string, node: string, opts?: { build?: boolean }): Promise<void> {
+  await registerModule(module, resolve(catalogDir, module, "module.json"));
+  if (opts?.build !== false) {
+    const built = await mesh(
+      `build ${forgeUrl("mesh-catalog")} --path modules/${module} --ref ${refFor("mesh-catalog")} --wait 1200s`,
+      1_500_000);
+    assert.doesNotMatch(built, /failed/i, built);
+  }
+  const manifest = (await on(CONTROL, `docker exec mesh-controller cat /${module}.json`)).out;
+  if (manifest.includes("MESH_BROKER_FILE")) await mesh(`module issue ${module} --node ${node}`);
+  await mesh(`assign ${node} ${module}`);
+}
+
+async function waitForContainer(node: string, container: string, seconds = 300): Promise<string> {
+  const deadline = Date.now() + seconds * 1_000;
+  let last = "";
+  while (Date.now() < deadline) {
+    const ps = (await on(node, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out;
+    last = ps;
+    const line = ps.split("\n").find((l) => l.split("\t")[0]?.trim() === container);
+    if (line && /^Up /.test(line.split("\t")[1]?.trim() ?? "")) return ps;
+    await new Promise((r) => setTimeout(r, 5_000));
+  }
+  const logs = (await on(node, `docker logs --tail 20 ${container} 2>&1`)).out;
+  throw new Error(`${container} never came up on ${node}:\n${last}\n--- logs ---\n${logs}`);
+}
+
+before(async () => {
+  if (skip) return;
+  const bed = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
+    onProgress: (m) => console.log(`raise: ${m}`),
+  });
+  instanceId = bed.instanceId;
+  console.log(`INSTANCE ${instanceId}${KEEP ? " (KEEP)" : ""}`);
+  console.log("NOTHING WAS LOADED: this scenario names no images; the mesh builds or pulls everything.");
+
+  // Genesis: the installer raises anchor into a mesh of one, BUILDING the control plane, and
+  // leaves the anchor enrolled with an agent running (hostService).
+  raised = await genesis({
+    instanceId,
+    node: CONTROL,
+    installer: installer as string,
+    catalogDir,
+    bundleTemplate: foundationBundle(bundle, []),
+    registry: REGISTRY,
+    source,
+    sourceRef,
+    toolsSource: forgeUrl(BASE.repo),
+    toolsRef: refFor(BASE.repo),
+    catalogSource: forgeUrl("mesh-catalog"),
+    catalogRef: refFor("mesh-catalog"),
+    sdkSource: forgeUrl("mesh-sdk"),
+    sdkRef: refFor("mesh-sdk"),
+    site: "hosting",
+    hostService: true,
+    hostBinary: binary,
+    log: (m) => console.log(m),
+  });
+  if (!raised.ok) throw new Error(`${raised.step || "genesis"}: ${raised.why}\n\n${raised.report.join("\n")}`);
+
+  // node2 joins the mesh: a token against the anchor's public broker endpoint, then an agent.
+  await mesh(`node add ${NODE}`);
+  const token = tokenFrom(await mesh(`token issue --node ${NODE}`));
+  const said = await must(NODE, `${HOST_PATH} enrol --token ${quote(token)}`);
+  assert.match(said, new RegExp(`enrolled as ${NODE}`), said);
+  await must(NODE, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
+}, { timeout: 3_000_000 });
+
+after(async () => {
+  if (KEEP) { console.log(`MESH_LAB_KEEP set — leaving ${instanceId} standing`); return; }
+  if (instanceId) await destroy(instanceId);
+  await destroyAll(`${SCENARIO}-`);
+}, { timeout: 900_000 });
+
+test("a joined node's consumers open the store and broker the mesh built and adopted, over the overlay", {
+  skip, timeout: 3_600_000,
+}, async () => {
+  // The overlay, so bindings carry `.internal` names; the firewall, so from:mesh is what admits them.
+  await mesh(`overlay place ${CONTROL} --hub --endpoint ${ANCHOR}:51820 --site lab`);
+  await mesh(`overlay place ${NODE} --site lab`);
+  await mesh(`assign ${CONTROL} networking`);
+  await mesh(`assign ${NODE} networking`);
+
+  // The shared base first — every module with code of its own stands on it.
+  await registerModule(BASE.module, baseManifest);
+  const base = await mesh(`build ${forgeUrl(BASE.repo)} --ref ${refFor(BASE.repo)} --wait 1200s`, 1_500_000);
+  assert.doesNotMatch(base, /failed/i, base);
+
+  // Adopt the foundation store and broker as the postgres and lavinmq modules, BUILT by the mesh.
+  // The store's superuser is the foundation's, made at genesis — carried in through `secret accept`
+  // before the push, or the module mints one the running store does not know.
+  await buildAndAssign("nftables", CONTROL, { build: false });
+  await buildAndAssign("postgres", CONTROL);
+  const superPw = (await must(CONTROL,
+    `docker inspect mesh-store --format '{{range .Config.Env}}{{println .}}{{end}}' | sed -n 's/^POSTGRES_PASSWORD=//p'`)).trim();
+  await must(CONTROL, `printf %s ${quote(superPw)} > /tmp/superuser && docker cp /tmp/superuser mesh-controller:/superuser`);
+  await mesh(`secret accept ${CONTROL} postgres superuser --from /superuser`);
+  await buildAndAssign("lavinmq", CONTROL);
+  await mesh(`push ${CONTROL}`, 600_000);
+  await waitForContainer(CONTROL, "mesh-postgres", 600);
+  await waitForContainer(CONTROL, "mesh-lavinmq", 600);
+
+  // The consumers on the joined node — built by the mesh, delivered from its registry.
+  await buildAndAssign("amqp-ping", NODE);
+  await buildAndAssign("letta", NODE);
+  await mesh(`push ${NODE}`, 900_000);
+
+  // 057: the provider node is composed again so its provisioners learn of the remote consumers.
+  await mesh(`push ${CONTROL}`, 600_000);
+
+  // THE BROKER, cross-node: amqp-ping's binding names the control-node's overlay name, its vhost is
+  // minted on the one broker, and it holds the connection.
+  await waitForContainer(NODE, "amqp-ping", 600);
+  const amqpBound = (await on(NODE, `cat /var/lib/amqp-ping/amqp.json 2>&1`)).out;
+  assert.match(amqpBound, new RegExp(`${CONTROL}\\.internal`),
+    `the amqp grant does not point at the control-node over the overlay:\n${amqpBound}`);
+  {
+    const deadline = Date.now() + 240_000;
+    let vhosts = "";
+    while (Date.now() < deadline) {
+      vhosts = (await on(CONTROL, `docker exec mesh-broker lavinmqctl list_vhosts 2>&1`)).out;
+      if (new RegExp(`${NODE}|amqp-ping`).test(vhosts)) break;
+      await new Promise((r) => setTimeout(r, 5_000));
+    }
+    assert.match(vhosts, new RegExp(`${NODE}|amqp-ping`),
+      `no vhost was minted on the one broker for the joined consumer:\n${vhosts}\n` +
+      `--- provisioner ---\n${(await on(CONTROL, `docker logs mesh-lavinmq 2>&1 | tail -20`)).out}\n` +
+      `--- consumer ---\n${(await on(NODE, `docker logs amqp-ping 2>&1 | tail -20`)).out}`);
+  }
+
+  // THE STORE, cross-node: letta's binding names the control-node, and the login the mesh derived
+  // authenticates on the one store with the password the provisioner minted. (letta's own app needs
+  // pgvector and is not the claim here — the credential reaching a real database is.)
+  const bound = await (async () => {
+    const deadline = Date.now() + 240_000;
+    let raw = "";
+    while (Date.now() < deadline) {
+      const got = await on(NODE, `cat /var/lib/letta/database.json 2>/dev/null`);
+      if (got.ok && /"as"/.test(got.out)) { raw = got.out; break; }
+      await new Promise((r) => setTimeout(r, 5_000));
+    }
+    assert.match(raw, /"as"/,
+      `the mesh never wrote letta's database binding:\n${raw}\n` +
+      `--- store provisioner ---\n${(await on(CONTROL, `docker logs mesh-postgres 2>&1 | tail -20`)).out}`);
+    return JSON.parse(raw) as { as: string; at: string; provision: string };
+  })();
+  assert.equal(bound.provision, "postgres-database", `letta was bound the wrong provision: ${bound.provision}`);
+  assert.match(bound.at, new RegExp(`${CONTROL}\\.internal`),
+    `letta's database binding does not point at the control-node over the overlay: ${bound.at}`);
+  const pw = (await must(NODE, `cat /var/lib/letta/database.secret`)).trim();
+  assert.ok(bound.as && pw, `letta's login or password was empty (as=${bound.as})`);
+  {
+    const conn = `postgresql://${bound.as}:${encodeURIComponent(pw)}@127.0.0.1:5432/${bound.as}?sslmode=disable`;
+    let pg = { out: "", ok: false };
+    const deadline = Date.now() + 120_000;
+    while (Date.now() < deadline) {
+      pg = await on(CONTROL, `docker exec mesh-postgres psql ${quote(conn)} -tAc 'select 1' 2>&1`);
+      if (pg.ok && /^1$/m.test(pg.out)) break;
+      if (/authentication failed/i.test(pg.out)) break;
+      await new Promise((r) => setTimeout(r, 5_000));
+    }
+    assert.doesNotMatch(pg.out, /authentication failed/i,
+      `the store does not know the password the mesh delivered letta:\n${pg.out}`);
+    assert.match(pg.out, /^1$/m, `letta's login could not open its database on the one store:\n${pg.out}`);
+  }
+
+  return `amqp: ${amqpBound.trim().slice(0, 120)}…\ndb: as=${bound.as} at=${bound.at}`;
+});