141 lines
6.9 KiB
TypeScript
141 lines
6.9 KiB
TypeScript
// Whether mail flows (novox/hq issue 690): the health tool is unhealthy, in words, when a message waits in
|
|
// postfix's deferred queue longer than the bound, when postfix does not run, when a public name does not
|
|
// resolve, and when the container cannot be reached — and healthy otherwise. Run against the compiled
|
|
// module, the way the runtime loads it.
|
|
|
|
import { test } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
|
|
import { mailFlows, DEFERRED_BOUND_SECONDS } from "../dist/health.js";
|
|
|
|
const NOW = 1_800_000_000;
|
|
|
|
type Answers = { status?: string | Error; queue?: string | Error; resolve?: string | Error };
|
|
|
|
/** A container that answers each command as told; an Error is a failed exec, with its stderr. */
|
|
function container(a: Answers) {
|
|
const asked: string[][] = [];
|
|
const exec = async (_name: string, command: string[]) => {
|
|
asked.push(command);
|
|
const answer = command[0] === "postfix" ? a.status ?? "" : command[0] === "postqueue" ? a.queue ?? "" : a.resolve ?? "";
|
|
if (answer instanceof Error) throw answer;
|
|
return answer;
|
|
};
|
|
return { exec, asked };
|
|
}
|
|
|
|
function failed(stderr: string): Error {
|
|
return Object.assign(new Error("Command failed: docker exec"), { stderr });
|
|
}
|
|
|
|
function deferred(id: string, ageSeconds: number, reason: string, to = "friend@example.com"): string {
|
|
return JSON.stringify({
|
|
queue_name: "deferred",
|
|
queue_id: id,
|
|
arrival_time: NOW - ageSeconds,
|
|
message_size: 1234,
|
|
sender: "someone@example.org",
|
|
recipients: [{ address: to, delay_reason: reason }],
|
|
});
|
|
}
|
|
|
|
test("an empty queue is healthy, and says so", async () => {
|
|
const c = container({ queue: "" });
|
|
const a = await mailFlows(c.exec, "mailu-smtp", NOW);
|
|
assert.equal(a.state, "healthy");
|
|
assert.equal(a.healthy, true);
|
|
assert.match(a.why, /deferred queue is empty/);
|
|
assert.deepEqual(a.queue, { messages: 0, deferred: 0, overdue: 0 });
|
|
assert.equal(c.asked.length, 3, "postfix, its queue and a lookup were each asked");
|
|
});
|
|
|
|
test("a recipient overdue on a name-service failure is unhealthy, with counts, its age and its reason", async () => {
|
|
const reason = "Host or domain name not found. Name service error for name=gmail.com type=MX: Host not found, try again";
|
|
const queue = [
|
|
deferred("AAA111", 3 * 86400 + 2 * 3600, reason),
|
|
deferred("BBB222", 5 * 60, "connect to mx.example.net[192.0.2.1]:25: Connection timed out"),
|
|
JSON.stringify({ queue_name: "active", queue_id: "CCC333", arrival_time: NOW - 10, recipients: [] }),
|
|
].join("\n");
|
|
const a = await mailFlows(container({ queue }).exec, "mailu-smtp", NOW);
|
|
assert.equal(a.state, "unhealthy");
|
|
assert.equal(a.healthy, false);
|
|
assert.match(a.why, /^Mail does not flow\./);
|
|
assert.match(a.why, /1 deferred recipient has waited longer than 30 minutes on a name-service failure/);
|
|
assert.match(a.why, /3 days 2 hours old to friend@example\.com/);
|
|
assert.ok(a.why.includes(reason), "the reason is said");
|
|
assert.deepEqual(a.queue, {
|
|
messages: 3, deferred: 2, overdue: 1, oldestDeferredSeconds: 3 * 86400 + 2 * 3600,
|
|
overdueDomains: ["example.com"], nameServiceFailures: 1,
|
|
});
|
|
});
|
|
|
|
test("one dead far domain overdue is healthy, and said", async () => {
|
|
const queue = [
|
|
deferred("AAA111", 2 * 86400, "connect to mx.dead.example[192.0.2.9]:25: Connection timed out", "a@dead.example"),
|
|
deferred("BBB222", 3600, "connect to mx.dead.example[192.0.2.9]:25: Connection timed out", "b@dead.example"),
|
|
].join("\n");
|
|
const a = await mailFlows(container({ queue }).exec, "mailu-smtp", NOW);
|
|
assert.equal(a.state, "healthy");
|
|
assert.equal(a.healthy, true);
|
|
assert.match(a.why, /2 deferred messages have waited longer than 30 minutes, only to dead\.example, which reads as the far side's/);
|
|
assert.match(a.why, /2 days old to a@dead\.example, waits because: connect to mx\.dead\.example/);
|
|
assert.deepEqual(a.queue?.overdueDomains, ["dead.example"]);
|
|
});
|
|
|
|
test("overdue mail to three domains is unhealthy, whatever the reason", async () => {
|
|
const timedOut = "connect to mx[192.0.2.1]:25: Connection timed out";
|
|
const queue = [
|
|
deferred("AAA111", 7200, timedOut, "a@one.example"),
|
|
deferred("BBB222", 3600, timedOut, "b@two.example"),
|
|
deferred("CCC333", 3600, timedOut, "c@three.example"),
|
|
].join("\n");
|
|
const a = await mailFlows(container({ queue }).exec, "mailu-smtp", NOW);
|
|
assert.equal(a.state, "unhealthy");
|
|
assert.match(a.why, /3 of 3 deferred messages have waited longer than 30 minutes, to 3 domains \(one\.example, two\.example, three\.example\)/);
|
|
assert.match(a.why, /the oldest, 2 hours old to a@one\.example/);
|
|
});
|
|
|
|
test("a deferred message younger than the bound is healthy, and is said", async () => {
|
|
const queue = deferred("BBB222", DEFERRED_BOUND_SECONDS - 60, "connect to mx.example.net: Connection timed out");
|
|
const a = await mailFlows(container({ queue }).exec, "mailu-smtp", NOW);
|
|
assert.equal(a.state, "healthy");
|
|
assert.equal(a.healthy, true);
|
|
assert.match(a.why, /1 deferred, the oldest 29 minutes old, within 30 minutes/);
|
|
});
|
|
|
|
test("a container that cannot be reached is unhealthy, and says what docker said", async () => {
|
|
const gone = failed("Error response from daemon: No such container: mailu-smtp\n");
|
|
const a = await mailFlows(container({ status: gone, queue: gone, resolve: gone }).exec, "mailu-smtp", NOW);
|
|
assert.equal(a.state, "unhealthy");
|
|
assert.equal(a.healthy, false);
|
|
assert.match(a.why, /postfix in mailu-smtp does not run: Error response from daemon: No such container: mailu-smtp\./);
|
|
assert.match(a.why, /postfix's queue in mailu-smtp could not be read: Error response from daemon: No such container/);
|
|
assert.equal(a.queue, undefined);
|
|
});
|
|
|
|
test("a name that does not resolve is unhealthy, though the queue is empty", async () => {
|
|
const a = await mailFlows(
|
|
container({ queue: "", resolve: failed("gaierror: [Errno -3] Try again\n") }).exec,
|
|
"mailu-smtp",
|
|
NOW,
|
|
);
|
|
assert.equal(a.state, "unhealthy");
|
|
assert.match(a.why, /mailu-smtp cannot resolve gmail-smtp-in\.l\.google\.com through its resolver: gaierror: \[Errno -3\] Try again\./);
|
|
});
|
|
|
|
test("a container that does not answer in time is said so", async () => {
|
|
const hung = Object.assign(new Error("Command failed"), { killed: true, signal: "SIGTERM" });
|
|
const a = await mailFlows(container({ queue: hung }).exec, "mailu-smtp", NOW);
|
|
assert.equal(a.state, "unhealthy");
|
|
assert.match(a.why, /postfix's queue in mailu-smtp could not be read: it did not answer in time\./);
|
|
});
|
|
|
|
test("the health tool is served without the admin API, on the smtp container it is told", async () => {
|
|
const { collectTools } = await import("@novox/mesh-sdk/tools");
|
|
await import("../dist/tools/index.js");
|
|
const mailu = collectTools({ MESH_MAILU_SMTP_CONTAINER: "mailu-smtp" }).find((m) => m.module === "mailu");
|
|
const names = (mailu?.tools ?? []).map((t) => t.name);
|
|
assert.deepEqual(names, ["mailu_health"]);
|
|
assert.match(mailu!.tools[0].description, /inside mailu-smtp/);
|
|
});
|