diff --git a/test/integration/mesh.test.ts b/test/integration/mesh.test.ts index e4d2100..80b6660 100644 --- a/test/integration/mesh.test.ts +++ b/test/integration/mesh.test.ts @@ -104,6 +104,45 @@ async function mesh(command: string, timeoutMs?: number): Promise { return must("anchor", `docker exec mesh-control /mesh-control ${command}`, timeoutMs); } +/** + * Wait until a machine has actually applied what it was last sent. + * + * **`push` sends; it does not wait.** It prints "sent N resource(s)" and returns, and the machine + * applies afterwards. So asserting on what a machine is running immediately after a push is a race + * — and the one this fixes lost it silently: `status` still described the *previous* apply, so it + * reported nothing wrong while the containers from this declaration did not exist yet. + * + * Asked of the mesh rather than of the machine, and in its own terms. A node is settled when it is + * neither waiting for what it was sent nor wrong about what it applied — the same two questions + * `status` answers, read as JSON so a test is not parsing a report meant for a person. + * + * A machine that reports a failure ends this at once rather than on the timeout: it is not going + * to become right by being waited for, and a refusal read after two minutes of polling is the same + * refusal, later. + */ +async function settled(node: string, withinMs = 240_000): Promise { + const until = Date.now() + withinMs; + let last = ""; + while (Date.now() < until) { + const said = await mesh("status --json"); + const state = JSON.parse(said) as { + wrong: { node: string; outcome: string; refused?: string; + failed?: { id: string; error: string }[] }[]; + waiting: { node: string; never: boolean }[]; + }; + const bad = state.wrong.find((w) => w.node === node); + if (bad) { + const why = [bad.refused, ...(bad.failed ?? []).map((f) => `${f.id}: ${f.error}`)] + .filter(Boolean).join("\n "); + throw new Error(`${node} did not apply what it was sent (${bad.outcome}):\n ${why}`); + } + if (!state.waiting.some((w) => w.node === node)) return; + last = said; + await new Promise((r) => setTimeout(r, 2000)); + } + throw new Error(`${node} never caught up with what it was sent:\n${last}`); +} + /** * The bundle, with every image reference pointed at this scenario's registry. * @@ -1866,13 +1905,19 @@ test("the forge runs, on a database the mesh gave it", { skip, timeout: 900_000 // waited for a database role, so when the containers were never created at all it reported "no // login was created" — true, and silent about the reason. A push that was accepted and an apply // that worked are different facts, and the second is the one this depends on. - const state = await mesh("status"); - assert.doesNotMatch(state, /failed/i, - `the machine did not do what it was told:\n${state}\n\n` + - `${(await on("anchor", `tail -30 /var/log/mesh-host.log`)).out}`); + // + // And waited for, because `push` sends without waiting. Reading `status` the instant it returns + // describes the apply *before* this one, which is how this test came to report a missing + // container while insisting the machine was fine. + await settled("anchor"); const running = (await on("anchor", `docker ps -a --format '{{.Names}} {{.Status}}'`)).out; + // Named with what the mesh meant to send, not only with what the machine has. A container that + // is absent because the mesh never asked for it and one that is absent because the machine could + // not make it are the same sentence here and different faults entirely, and the plan is the only + // thing that tells them apart. assert.match(running, /\bpostgres\b/, `the database module was pushed and no container for it exists:\n${running}\n\n` + + `what the mesh would send anchor:\n${await mesh("plan anchor")}\n\n` + `${(await on("anchor", `tail -30 /var/log/mesh-host.log`)).out}`); // The database first: until the provisioner has made the login, the forge has nothing to