From 960fa3607f34d67424a0b4b079c988ea15750f21 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 21 Sep 2026 00:33:23 +0200 Subject: [PATCH] The genesis bed waits for the node to settle before asking it anything Applying the packet filter restarts the container runtime a few seconds after the installer's last push returns; a command racing that window dies with 'No such exec instance'. Wait for the node to report applied and current, and retry that error like the recreate it is. --- test/integration/one-node-mesh.test.ts | 40 +++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/test/integration/one-node-mesh.test.ts b/test/integration/one-node-mesh.test.ts index 354a966..503845a 100644 --- a/test/integration/one-node-mesh.test.ts +++ b/test/integration/one-node-mesh.test.ts @@ -231,7 +231,10 @@ async function mesh(command: string, timeoutMs?: number): Promise { for (;;) { const { out, ok } = await on(CONTROL, `docker exec mesh-controller /mesh-controller ${command}`, timeoutMs); if (ok) return out; - if (/is not running|No such container/i.test(out) && Date.now() < deadline) { + // "No such exec instance" and a daemon that cannot be reached are the same race one layer down: + // applying the packet filter restarts the container runtime itself, and every container with + // it, a few seconds after the installer's last push returns. + if (/is not running|No such container|No such exec instance|Cannot connect to the Docker daemon/i.test(out) && Date.now() < deadline) { await new Promise((r) => setTimeout(r, 2_000)); continue; } @@ -239,6 +242,34 @@ async function mesh(command: string, timeoutMs?: number): Promise { } } +/** Until the anchor reports the last declaration genesis pushed as applied and current. */ +async function settledAfterGenesis(withinMs = 300_000): Promise { + const deadline = Date.now() + withinMs; + let last = ""; + while (Date.now() < deadline) { + const asked = await on(CONTROL, `docker exec mesh-controller /mesh-controller status --json`); + if (asked.ok) { + try { + const state = JSON.parse(asked.out) as { + wrong: { node: string; outcome: string }[]; + waiting: { node: string }[]; + reported: { node: string; outcome: string; current: boolean }[]; + }; + const bad = state.wrong.find((w) => w.node === CONTROL); + if (bad) throw new Error(`${CONTROL} did not apply what genesis sent: ${bad.outcome}\n${asked.out}`); + const word = state.reported.find((r) => r.node === CONTROL); + if (!state.waiting.some((w) => w.node === CONTROL) && word?.outcome === "applied" && word.current) return; + last = asked.out; + } catch (err) { + if (err instanceof Error && err.message.includes("did not apply")) throw err; + last = asked.out; + } + } + await new Promise((r) => setTimeout(r, 5_000)); + } + throw new Error(`${CONTROL} never settled after genesis within ${Math.round(withinMs / 1000)}s. Last:\n${last}`); +} + /** * Stop the machine and start it again, the way a power cut or a kernel upgrade would. * @@ -519,6 +550,13 @@ before(async () => { // saying it published an image and the registry serving one are different facts, and it is the // second that matters. + // **The machine is still applying what genesis pushed last.** The installer's final pushes — the + // private network, the packet filter — return when the mesh has sent them, not when the host has + // finished; applying the filter restarts the container runtime, and every container with it. + // Asked of the mesh rather than slept through: nothing below is sent until the node reports the + // declaration it was given as applied and current. + await settledAfterGenesis(); + await step("R2", FOUNDATION, GENESIS, async () => { await waitForContainer(CONTROL, "mesh-store", 120); await waitForContainer(CONTROL, "mesh-broker", 120);