The genesis bed waits for the node to settle before asking it anything

Applying the packet filter restarts the container runtime a few seconds
after the installer's last push returns; a command racing that window dies
with 'No such exec instance'. Wait for the node to report applied and
current, and retry that error like the recreate it is.
This commit is contained in:
2026-09-21 00:33:23 +02:00
parent 9da2d01ca0
commit 960fa3607f
+39 -1
View File
@@ -231,7 +231,10 @@ async function mesh(command: string, timeoutMs?: number): Promise<string> {
for (;;) { for (;;) {
const { out, ok } = await on(CONTROL, `docker exec mesh-controller /mesh-controller ${command}`, timeoutMs); const { out, ok } = await on(CONTROL, `docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
if (ok) return out; if (ok) return out;
if (/is not running|No such container/i.test(out) && Date.now() < deadline) { // "No such exec instance" and a daemon that cannot be reached are the same race one layer down:
// applying the packet filter restarts the container runtime itself, and every container with
// it, a few seconds after the installer's last push returns.
if (/is not running|No such container|No such exec instance|Cannot connect to the Docker daemon/i.test(out) && Date.now() < deadline) {
await new Promise((r) => setTimeout(r, 2_000)); await new Promise((r) => setTimeout(r, 2_000));
continue; continue;
} }
@@ -239,6 +242,34 @@ async function mesh(command: string, timeoutMs?: number): Promise<string> {
} }
} }
/** Until the anchor reports the last declaration genesis pushed as applied and current. */
async function settledAfterGenesis(withinMs = 300_000): Promise<void> {
const deadline = Date.now() + withinMs;
let last = "";
while (Date.now() < deadline) {
const asked = await on(CONTROL, `docker exec mesh-controller /mesh-controller status --json`);
if (asked.ok) {
try {
const state = JSON.parse(asked.out) as {
wrong: { node: string; outcome: string }[];
waiting: { node: string }[];
reported: { node: string; outcome: string; current: boolean }[];
};
const bad = state.wrong.find((w) => w.node === CONTROL);
if (bad) throw new Error(`${CONTROL} did not apply what genesis sent: ${bad.outcome}\n${asked.out}`);
const word = state.reported.find((r) => r.node === CONTROL);
if (!state.waiting.some((w) => w.node === CONTROL) && word?.outcome === "applied" && word.current) return;
last = asked.out;
} catch (err) {
if (err instanceof Error && err.message.includes("did not apply")) throw err;
last = asked.out;
}
}
await new Promise((r) => setTimeout(r, 5_000));
}
throw new Error(`${CONTROL} never settled after genesis within ${Math.round(withinMs / 1000)}s. Last:\n${last}`);
}
/** /**
* Stop the machine and start it again, the way a power cut or a kernel upgrade would. * Stop the machine and start it again, the way a power cut or a kernel upgrade would.
* *
@@ -519,6 +550,13 @@ before(async () => {
// saying it published an image and the registry serving one are different facts, and it is the // saying it published an image and the registry serving one are different facts, and it is the
// second that matters. // second that matters.
// **The machine is still applying what genesis pushed last.** The installer's final pushes — the
// private network, the packet filter — return when the mesh has sent them, not when the host has
// finished; applying the filter restarts the container runtime, and every container with it.
// Asked of the mesh rather than slept through: nothing below is sent until the node reports the
// declaration it was given as applied and current.
await settledAfterGenesis();
await step("R2", FOUNDATION, GENESIS, async () => { await step("R2", FOUNDATION, GENESIS, async () => {
await waitForContainer(CONTROL, "mesh-store", 120); await waitForContainer(CONTROL, "mesh-store", 120);
await waitForContainer(CONTROL, "mesh-broker", 120); await waitForContainer(CONTROL, "mesh-broker", 120);