diff --git a/test/integration/built-store-cross-node.test.ts b/test/integration/built-store-cross-node.test.ts index b323578..d69c605 100644 --- a/test/integration/built-store-cross-node.test.ts +++ b/test/integration/built-store-cross-node.test.ts @@ -125,7 +125,7 @@ async function buildAndAssign(module: string, node: string, opts?: { build?: boo 1_500_000); assert.doesNotMatch(built, /failed/i, built); } - const manifest = (await on(CONTROL, `docker exec mesh-controller cat /${module}.json`)).out; + const manifest = readFileSync(resolve(catalogDir, module, "module.json"), "utf8"); if (manifest.includes("MESH_BROKER_FILE")) { const issued = await mesh(`module issue ${module} --node ${node}`); assert.match(issued, /scoped to what it (emits and consumes|consumes and emits)/, @@ -188,7 +188,7 @@ before(async () => { const said = await must(NODE, `${HOST_PATH} enrol --token ${quote(token)}`); assert.match(said, new RegExp(`enrolled as ${NODE}`), said); await must(NODE, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`); -}, { timeout: 3_000_000 }); +}, { timeout: 7_200_000 }); after(async () => { if (KEEP) { console.log(`MESH_LAB_KEEP set — leaving ${instanceId} standing`); return; } @@ -200,8 +200,8 @@ test("a joined node's consumers open the store and broker the mesh built and ado skip, timeout: 3_600_000, }, async () => { // The overlay, so bindings carry `.internal` names; the firewall, so from:mesh is what admits them. - await mesh(`overlay place ${CONTROL} --hub --endpoint ${ANCHOR}:51820 --site lab`); - await mesh(`overlay place ${NODE} --site lab`); + await mesh(`overlay place ${CONTROL} --hub --endpoint ${ANCHOR}:51820 --site hosting`); + await mesh(`overlay place ${NODE} --site hosting`); await mesh(`assign ${CONTROL} networking`); await mesh(`assign ${NODE} networking`); @@ -217,17 +217,26 @@ test("a joined node's consumers open the store and broker the mesh built and ado await buildAndAssign("lavinmq", CONTROL); await waitForContainer(CONTROL, "mesh-lavinmq", 600); - // The consumers on the joined node — built by the mesh, delivered from its registry. + // The consumer on the joined node — built by the mesh, delivered from its registry. One consumer, + // amqp-ping, because it is one of the eight modules the mesh can actually build; the catalogue's + // DB consumers all lack a build section (hq issue 060), so the store's cross-node proof stays with + // the stocked-image bed until one of them gains one. NOTE: this delivery is the path hq issues + // 042 (a node has no registry account) and 048 (nothing makes a machine trust the registry) leave + // open — this bed is their honest gate, red until they are fixed. await buildAndAssign("amqp-ping", NODE); - await buildAndAssign("letta", NODE); - await mesh(`push ${NODE}`, 900_000); // 057: the provider node is composed again so its provisioners learn of the remote consumers. await mesh(`push ${CONTROL}`, 600_000); // THE BROKER, cross-node: amqp-ping's binding names the control-node's overlay name, its vhost is // minted on the one broker, and it holds the connection. - await waitForContainer(NODE, "amqp-ping", 600); + try { + await waitForContainer(NODE, "amqp-ping", 600); + } catch (err) { + const hostLog = (await on(NODE, `tail -40 /var/log/mesh-host.log`)).out; + throw new Error(`${(err as Error).message}\n--- ${NODE} mesh-host.log (a pull that failed ` + + `never reaches container logs; hq issues 042/048) ---\n${hostLog}`); + } const amqpBound = (await on(NODE, `cat /var/lib/amqp-ping/amqp.json 2>&1`)).out; assert.match(amqpBound, new RegExp(`${CONTROL}\\.internal`), `the amqp grant does not point at the control-node over the overlay:\n${amqpBound}`); @@ -245,41 +254,15 @@ test("a joined node's consumers open the store and broker the mesh built and ado `--- consumer ---\n${(await on(NODE, `docker logs amqp-ping 2>&1 | tail -20`)).out}`); } - // THE STORE, cross-node: letta's binding names the control-node, and the login the mesh derived - // authenticates on the one store with the password the provisioner minted. (letta's own app needs - // pgvector and is not the claim here — the credential reaching a real database is.) - const bound = await (async () => { - const deadline = Date.now() + 240_000; - let raw = ""; - while (Date.now() < deadline) { - const got = await on(NODE, `cat /var/lib/letta/database.json 2>/dev/null`); - if (got.ok && /"as"/.test(got.out)) { raw = got.out; break; } - await new Promise((r) => setTimeout(r, 5_000)); - } - assert.match(raw, /"as"/, - `the mesh never wrote letta's database binding:\n${raw}\n` + - `--- store provisioner ---\n${(await on(CONTROL, `docker logs mesh-postgres 2>&1 | tail -20`)).out}`); - return JSON.parse(raw) as { as: string; at: string; provision: string }; - })(); - assert.equal(bound.provision, "postgres-database", `letta was bound the wrong provision: ${bound.provision}`); - assert.match(bound.at, new RegExp(`${CONTROL}\\.internal`), - `letta's database binding does not point at the control-node over the overlay: ${bound.at}`); - const pw = (await must(NODE, `cat /var/lib/letta/database.secret`)).trim(); - assert.ok(bound.as && pw, `letta's login or password was empty (as=${bound.as})`); - { - const conn = `postgresql://${bound.as}:${encodeURIComponent(pw)}@127.0.0.1:5432/${bound.as}?sslmode=disable`; - let pg = { out: "", ok: false }; - const deadline = Date.now() + 120_000; - while (Date.now() < deadline) { - pg = await on(CONTROL, `docker exec mesh-postgres psql ${quote(conn)} -tAc 'select 1' 2>&1`); - if (pg.ok && /^1$/m.test(pg.out)) break; - if (/authentication failed/i.test(pg.out)) break; - await new Promise((r) => setTimeout(r, 5_000)); - } - assert.doesNotMatch(pg.out, /authentication failed/i, - `the store does not know the password the mesh delivered letta:\n${pg.out}`); - assert.match(pg.out, /^1$/m, `letta's login could not open its database on the one store:\n${pg.out}`); + // Adoption did not cost the foundation: the containers genesis raised are still the ones running. + const up = await must(CONTROL, `docker ps --format '{{.Names}}'`); + for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) { + assert.match(up, new RegExp(`(^|\\n)${c}(\\n|$)`), `${c} did not survive adoption:\n${up}`); } + // Steady a moment, then confirm the consumer did not crash-loop after connecting. + await new Promise((r) => setTimeout(r, 15_000)); + assert.match((await on(NODE, `docker ps --format '{{.Names}}\t{{.Status}}'`)).out, + /amqp-ping\tUp/, `amqp-ping did not stay up on ${NODE}`); - return `amqp: ${amqpBound.trim().slice(0, 120)}…\ndb: as=${bound.as} at=${bound.at}`; + return `amqp: ${amqpBound.trim().slice(0, 160)}`; });