Merge pull request 'The settle check records settling instead of inferring it from the clock' (#32) from fix/settle-check-edge into main
This commit was merged in pull request #32.
This commit is contained in:
@@ -494,14 +494,14 @@ test("consumers on a joined node get their databases from the one foundation sto
|
|||||||
const stable = expected.filter((n) => n !== "letta");
|
const stable = expected.filter((n) => n !== "letta");
|
||||||
for (const name of stable) {
|
for (const name of stable) {
|
||||||
const deadline = Date.now() + 300_000;
|
const deadline = Date.now() + 300_000;
|
||||||
let prev = -1, stableSince = 0, last = "?";
|
let prev = -1, stableSince = 0, last = "?", settled = false;
|
||||||
while (Date.now() < deadline) {
|
while (Date.now() < deadline) {
|
||||||
const [running, count] = (await must(NODE, `docker inspect -f '{{.State.Running}} {{.RestartCount}}' ${name}`)).trim().split(" ");
|
const [running, count] = (await must(NODE, `docker inspect -f '{{.State.Running}} {{.RestartCount}}' ${name}`)).trim().split(" ");
|
||||||
last = `running=${running} restarts=${count}`;
|
last = `running=${running} restarts=${count}`;
|
||||||
const n = Number(count);
|
const n = Number(count);
|
||||||
if (running === "true" && n === prev) {
|
if (running === "true" && n === prev) {
|
||||||
if (stableSince === 0) stableSince = Date.now();
|
if (stableSince === 0) stableSince = Date.now();
|
||||||
if (Date.now() - stableSince >= 30_000) break; // up and unchanged for 30s — settled
|
if (Date.now() - stableSince >= 30_000) { settled = true; break; } // up and unchanged 30s
|
||||||
} else {
|
} else {
|
||||||
prev = n; stableSince = 0;
|
prev = n; stableSince = 0;
|
||||||
}
|
}
|
||||||
@@ -510,7 +510,7 @@ test("consumers on a joined node get their databases from the one foundation sto
|
|||||||
const [running, count] = (await must(NODE, `docker inspect -f '{{.State.Running}} {{.RestartCount}}' ${name}`)).trim().split(" ");
|
const [running, count] = (await must(NODE, `docker inspect -f '{{.State.Running}} {{.RestartCount}}' ${name}`)).trim().split(" ");
|
||||||
assert.equal(running, "true",
|
assert.equal(running, "true",
|
||||||
`${name} is not running after the push (${last}):\n${(await on(NODE, `docker logs ${name} 2>&1 | tail -40`)).out}`);
|
`${name} is not running after the push (${last}):\n${(await on(NODE, `docker logs ${name} 2>&1 | tail -40`)).out}`);
|
||||||
if (Date.now() >= deadline)
|
if (!settled)
|
||||||
assert.fail(`${name} never stopped restarting within 300s (last ${last}) — a crash-loop, not startup churn:\n${(await on(NODE, `docker logs ${name} 2>&1 | tail -40`)).out}`);
|
assert.fail(`${name} never stopped restarting within 300s (last ${last}) — a crash-loop, not startup churn:\n${(await on(NODE, `docker logs ${name} 2>&1 | tail -40`)).out}`);
|
||||||
void count;
|
void count;
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user