diff --git a/test/integration/fresh-mesh.test.ts b/test/integration/fresh-mesh.test.ts index 7b95aa5..83c83fe 100644 --- a/test/integration/fresh-mesh.test.ts +++ b/test/integration/fresh-mesh.test.ts @@ -162,6 +162,29 @@ async function mesh(command: string, timeoutMs?: number): Promise { return must(CONTROL, `docker exec mesh-control /mesh-control ${command}`, timeoutMs); } +/** + * Wait for one container, BY NAME, to be running. + * + * **Named exactly, because substring matching passed a step that had failed.** Waiting for "a + * container whose name contains lavinmq" was satisfied by the broker — `lavinmq`, up and healthy — + * while the thing actually under test, the module's own runtime `mesh-lavinmq`, was crash-looping + * beside it. The step went green and the fault was found by reading `docker ps` by hand. + */ +async function waitForContainer(node: string, container: string, seconds = 200): Promise { + const deadline = Date.now() + seconds * 1_000; + let last = ""; + while (Date.now() < deadline) { + const ps = (await on(node, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out; + last = ps; + const line = ps.split("\n").find((l) => l.split("\t")[0]?.trim() === container); + if (line && /^Up /.test(line.split("\t")[1]?.trim() ?? "")) return ps; + await new Promise((r) => setTimeout(r, 5_000)); + } + const logs = (await on(node, `docker logs --tail 15 ${container} 2>&1`)).out; + throw new Error( + `${container} is not running on ${node}.\n\ncontainers:\n${last}\n\nwhat it said:\n${logs}`); +} + /** * Register a module from a manifest on this workstation. * @@ -346,15 +369,15 @@ before(async () => { 1_500_000); assert.doesNotMatch(built, /failed/i, built); await mesh(`assign ${CONTROL} ${PROVIDER.module}`); + // **Its account on the mesh's own broker, and not swallowed.** A module's runtime is a tool + // host: it connects to the mesh broker before it does anything, and what it reads is a sealed + // credential document. Without an account the mesh still fills the secret this module declares + // it owns — with a generated value — so the container starts, fails to parse a password as a + // credential, and crash-loops on a JSON syntax error that says nothing about the missing + // account. Issuing it is not optional and neither is hearing that it failed. + await mesh(`module issue ${PROVIDER.module} --node ${CONTROL}`); await mesh(`push ${CONTROL}`, 600_000); - for (let i = 0; i < 60; i++) { - const ps = (await on(CONTROL, `docker ps --format '{{.Names}}\t{{.Status}}'`)).out; - const line = ps.split("\n").find((l) => l.includes(PROVIDER.module)); - if (line && /Up /.test(line)) return ps; - await new Promise((r) => setTimeout(r, 5_000)); - } - throw new Error(`${PROVIDER.module} never came up on ${CONTROL}:\n` + - (await on(CONTROL, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out); + return waitForContainer(CONTROL, "mesh-lavinmq"); }); // ---- 5. THE ANCHOR RUNS IT ------------------------------------------------------------------ @@ -362,18 +385,10 @@ before(async () => { // The machine that built it. This is the case every earlier proof covered, and it is here as the // control for step 6: if this fails, step 6's failure says nothing about fetching. await step("the anchor runs the module the mesh built", NEEDS, async () => { - await mesh(`module issue ${MODULE.module} --node ${CONTROL}`).catch(() => {}); + await mesh(`module issue ${MODULE.module} --node ${CONTROL}`); await mesh(`assign ${CONTROL} ${MODULE.module}`); await mesh(`push ${CONTROL}`, 600_000); - for (let i = 0; i < 40; i++) { - const ps = (await on(CONTROL, `docker ps --format '{{.Names}}\t{{.Status}}'`)).out; - if (/amqp-ping/.test(ps) && /Up /.test(ps.split("\n").find((l) => l.includes("amqp-ping")) ?? "")) { - return ps; - } - await new Promise((r) => setTimeout(r, 5_000)); - } - throw new Error(`amqp-ping never came up on ${CONTROL}:\n` + - (await on(CONTROL, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out); + return waitForContainer(CONTROL, MODULE.module); }); // ---- 6. AND A MACHINE THAT DID NOT BUILD IT ------------------------------------------------- @@ -387,20 +402,15 @@ before(async () => { // evidence of what does work. await step(`a joined machine runs the module the mesh built`, "the anchor runs the module the mesh built", async () => { - await mesh(`module issue ${MODULE.module} --node ${SECOND}`).catch(() => {}); + await mesh(`module issue ${MODULE.module} --node ${SECOND}`); await mesh(`assign ${SECOND} ${MODULE.module}`); await mesh(`push ${SECOND}`, 600_000); - for (let i = 0; i < 40; i++) { - const ps = (await on(SECOND, `docker ps --format '{{.Names}}\t{{.Status}}'`)).out; - const line = ps.split("\n").find((l) => l.includes("amqp-ping")); - if (line && /Up /.test(line)) return ps; - await new Promise((r) => setTimeout(r, 5_000)); + try { + return await waitForContainer(SECOND, MODULE.module); + } catch (err) { + const log = (await on(SECOND, `tail -40 /var/log/mesh-host.log`)).out; + throw new Error(`${(err as Error).message}\n\nwhat the host said:\n${log}`); } - const state = (await on(SECOND, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out; - const log = (await on(SECOND, `tail -40 /var/log/mesh-host.log`)).out; - throw new Error( - `amqp-ping never came up on ${SECOND} — the machine that did NOT build it.\n\n` + - `containers:\n${state}\n\nwhat the host said:\n${log}`); }); console.log(`\n================ WHAT THIS MESH DID FOR ITSELF ================`);