Brake counts only successful re-applies; only a timeout ends a pass
A lost consumer whose re-apply fails is retried at the next check with its count unchanged, instead of waiting out a backoff meant for adapters whose create and holds disagree. A check that fails for one consumer no longer stops checking the consumers after it; only a timeout does.
This commit is contained in:
+67
-3
@@ -241,9 +241,9 @@ test("provisioner treats a failed or hung holds as could-not-ask, and never logs
|
||||
console.error = original;
|
||||
}
|
||||
assert.equal(creates, 2); // the first pass only; nothing is applied again on "could not ask"
|
||||
// After one consumer's check fails, the rest of that pass is not asked: "b" follows "a" and is
|
||||
// never reached, because every pass stops at "a".
|
||||
assert.ok(askedFor.length >= 2 && askedFor.every((as) => as === "a"), askedFor.join(","));
|
||||
// A failure that is not a timeout may be one consumer's alone: the consumers after it are still
|
||||
// asked on every pass.
|
||||
assert.ok(askedFor.filter((as) => as === "b").length >= 2, askedFor.join(","));
|
||||
assert.ok(logged.some((l) => l.includes("Command failed: tool --password ***")));
|
||||
assert.ok(!logged.some((l) => l.includes("s3cret-pw")));
|
||||
|
||||
@@ -272,6 +272,70 @@ test("provisioner treats a failed or hung holds as could-not-ask, and never logs
|
||||
assert.ok(hungLogged.some((l) => l.includes("no answer within 20ms")));
|
||||
});
|
||||
|
||||
test("provisioner stops a verify pass at a timeout, and retries a failed re-apply at the next check", async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), "prov-retry-"));
|
||||
await writeFile(join(dir, "a.secret"), "pa");
|
||||
await writeFile(join(dir, "b.secret"), "pb");
|
||||
const receives = join(dir, "cache.json");
|
||||
await writeFile(
|
||||
receives,
|
||||
JSON.stringify({ requirement: "cache", given: [{ as: "a", secret: join(dir, "a.secret") }, { as: "b", secret: join(dir, "b.secret") }] }),
|
||||
);
|
||||
const original = console.error;
|
||||
console.error = () => {};
|
||||
try {
|
||||
// A timeout on "a" means "b" is not asked in that pass.
|
||||
const askedFor: string[] = [];
|
||||
const stopHung = runProvisioner(
|
||||
"cache",
|
||||
{
|
||||
async create() {},
|
||||
async remove() {},
|
||||
holds(p) {
|
||||
askedFor.push(p.as);
|
||||
return p.as === "a" ? new Promise<boolean>(() => {}) : Promise.resolve(true);
|
||||
},
|
||||
},
|
||||
{ receives, everyMs: 5, verifyEveryMs: 30, holdsTimeoutMs: 10 },
|
||||
);
|
||||
await new Promise((r) => setTimeout(r, 150));
|
||||
stopHung();
|
||||
assert.ok(askedFor.length >= 2 && askedFor.every((as) => as === "a"), askedFor.join(","));
|
||||
|
||||
// A lost consumer whose re-apply fails is retried at the next check, not held back by the brake,
|
||||
// and the retry that succeeds counts as the first re-apply, not the second.
|
||||
let held = true;
|
||||
let createCalls = 0;
|
||||
const logged: string[] = [];
|
||||
console.error = (...a: unknown[]) => logged.push(a.join(" "));
|
||||
const stop = runProvisioner(
|
||||
"cache",
|
||||
{
|
||||
async create(p) {
|
||||
if (p.as !== "a") return;
|
||||
createCalls++;
|
||||
if (createCalls === 2) throw new Error("write refused"); // the first re-apply fails
|
||||
if (createCalls === 3) held = true; // the retry works
|
||||
},
|
||||
async remove() {},
|
||||
async holds(p) {
|
||||
return p.as === "a" ? held : true;
|
||||
},
|
||||
},
|
||||
{ receives, everyMs: 5, verifyEveryMs: 20 },
|
||||
);
|
||||
await waitFor(() => createCalls === 1, 2000);
|
||||
held = false; // the backend loses "a"
|
||||
await waitFor(() => createCalls === 3, 2000);
|
||||
await new Promise((r) => setTimeout(r, 100));
|
||||
stop();
|
||||
assert.equal(createCalls, 3);
|
||||
assert.ok(!logged.some((l) => l.includes("create does not produce what holds checks")), logged.join("\n"));
|
||||
} finally {
|
||||
console.error = original;
|
||||
}
|
||||
});
|
||||
|
||||
test("modules can SERVE: a real async tool, loaded and invoked over the broker", async () => {
|
||||
resetTools();
|
||||
|
||||
|
||||
Reference in New Issue
Block a user