From 48f8fe6168f79b7126b9684b513ef3c6137fcafb Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 17 Sep 2026 22:39:48 +0200 Subject: [PATCH 1/2] The settle check records settling instead of inferring it from the clock The review found a race: a container that settled in the window's last seconds could be re-inspected past the deadline and failed as 'never stopped restarting'. A boolean now says what happened. https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx --- test/integration/assigned-two-node-db.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/integration/assigned-two-node-db.test.ts b/test/integration/assigned-two-node-db.test.ts index e826863..b3b0ffd 100644 --- a/test/integration/assigned-two-node-db.test.ts +++ b/test/integration/assigned-two-node-db.test.ts @@ -500,14 +500,14 @@ test("consumers on a joined node get their databases from the one foundation sto const stable = expected.filter((n) => n !== "letta"); for (const name of stable) { const deadline = Date.now() + 300_000; - let prev = -1, stableSince = 0, last = "?"; + let prev = -1, stableSince = 0, last = "?", settled = false; while (Date.now() < deadline) { const [running, count] = (await must(NODE, `docker inspect -f '{{.State.Running}} {{.RestartCount}}' ${name}`)).trim().split(" "); last = `running=${running} restarts=${count}`; const n = Number(count); if (running === "true" && n === prev) { if (stableSince === 0) stableSince = Date.now(); - if (Date.now() - stableSince >= 30_000) break; // up and unchanged for 30s — settled + if (Date.now() - stableSince >= 30_000) { settled = true; break; } // up and unchanged 30s } else { prev = n; stableSince = 0; } @@ -516,7 +516,7 @@ test("consumers on a joined node get their databases from the one foundation sto const [running, count] = (await must(NODE, `docker inspect -f '{{.State.Running}} {{.RestartCount}}' ${name}`)).trim().split(" "); assert.equal(running, "true", `${name} is not running after the push (${last}):\n${(await on(NODE, `docker logs ${name} 2>&1 | tail -40`)).out}`); - if (Date.now() >= deadline) + if (!settled) assert.fail(`${name} never stopped restarting within 300s (last ${last}) — a crash-loop, not startup churn:\n${(await on(NODE, `docker logs ${name} 2>&1 | tail -40`)).out}`); void count; } From 8374f4dc19fa36ad0547628f4889501be4ceaeed Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 17 Sep 2026 23:07:50 +0200 Subject: [PATCH 2/2] warm refuses a departed instance instead of crashing into it ready() asked a gone instance for its snapshots before judge could say 'no longer standing'; a stale warm.json from any earlier scenario made every warm run fail in milliseconds. The question is now asked only of an instance that still exists. https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx --- src/warm.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/warm.ts b/src/warm.ts index 902a243..020c8e5 100644 --- a/src/warm.ts +++ b/src/warm.ts @@ -141,7 +141,11 @@ export async function ready( ): Promise { const warm = remembered(); const standing = await standingNow(); - const has = warm ? (await snapshots(warm.instanceId)).includes(label) : false; + // Asked only of an instance that still exists: snapshots() throws for one that is gone, and + // judge already has the words for that refusal — a remembered instance another scenario (or + // another month) left behind must be refused, not crashed into. + const has = warm && standing.includes(warm.instanceId) + ? (await snapshots(warm.instanceId)).includes(label) : false; return judge(warm, scenario, standing, has, whatWasTested(env)); }