mesh-catalog/modules/mailu/test/mail-flows.test.ts

// Whether mail flows (novox/hq issue 690): the health tool is unhealthy, in words, when a message waits in
// postfix's deferred queue longer than the bound, when postfix does not run, when a public name does not
// resolve, and when the container cannot be reached — and healthy otherwise. Run against the compiled
// module, the way the runtime loads it.

import { test } from "node:test";
import assert from "node:assert/strict";

import { mailFlows, DEFERRED_BOUND_SECONDS } from "../dist/health.js";

const NOW = 1_800_000_000;

type Answers = { status?: string | Error; queue?: string | Error; resolve?: string | Error };

/** A container that answers each command as told; an Error is a failed exec, with its stderr. */
function container(a: Answers) {
  const asked: string[][] = [];
  const exec = async (_name: string, command: string[]) => {
    asked.push(command);
    const answer = command[0] === "postfix" ? a.status ?? "" : command[0] === "postqueue" ? a.queue ?? "" : a.resolve ?? "";
    if (answer instanceof Error) throw answer;
    return answer;
  };
  return { exec, asked };
}

function failed(stderr: string): Error {
  return Object.assign(new Error("Command failed: docker exec"), { stderr });
}

function deferred(id: string, ageSeconds: number, reason: string, to = "friend@example.com"): string {
  return JSON.stringify({
    queue_name: "deferred",
    queue_id: id,
    arrival_time: NOW - ageSeconds,
    message_size: 1234,
    sender: "someone@example.org",
    recipients: [{ address: to, delay_reason: reason }],
  });
}

test("an empty queue is healthy, and says so", async () => {
  const c = container({ queue: "" });
  const a = await mailFlows(c.exec, "mailu-smtp", NOW);
  assert.equal(a.state, "healthy");
  assert.equal(a.healthy, true);
  assert.match(a.why, /deferred queue is empty/);
  assert.deepEqual(a.queue, { messages: 0, deferred: 0, overdue: 0 });
  assert.equal(c.asked.length, 3, "postfix, its queue and a lookup were each asked");
});

test("a recipient overdue on a name-service failure is unhealthy, with counts, its age and its reason", async () => {
  const reason = "Host or domain name not found. Name service error for name=gmail.com type=MX: Host not found, try again";
  const queue = [
    deferred("AAA111", 3 * 86400 + 2 * 3600, reason),
    deferred("BBB222", 5 * 60, "connect to mx.example.net[192.0.2.1]:25: Connection timed out"),
    JSON.stringify({ queue_name: "active", queue_id: "CCC333", arrival_time: NOW - 10, recipients: [] }),
  ].join("\n");
  const a = await mailFlows(container({ queue }).exec, "mailu-smtp", NOW);
  assert.equal(a.state, "unhealthy");
  assert.equal(a.healthy, false);
  assert.match(a.why, /^Mail does not flow\./);
  assert.match(a.why, /1 deferred recipient has waited longer than 30 minutes on a name-service failure/);
  assert.match(a.why, /3 days 2 hours old to friend@example\.com/);
  assert.ok(a.why.includes(reason), "the reason is said");
  assert.deepEqual(a.queue, {
    messages: 3, deferred: 2, overdue: 1, oldestDeferredSeconds: 3 * 86400 + 2 * 3600,
    overdueDomains: ["example.com"], nameServiceFailures: 1,
  });
});

test("one dead far domain overdue is healthy, and said", async () => {
  const queue = [
    deferred("AAA111", 2 * 86400, "connect to mx.dead.example[192.0.2.9]:25: Connection timed out", "a@dead.example"),
    deferred("BBB222", 3600, "connect to mx.dead.example[192.0.2.9]:25: Connection timed out", "b@dead.example"),
  ].join("\n");
  const a = await mailFlows(container({ queue }).exec, "mailu-smtp", NOW);
  assert.equal(a.state, "healthy");
  assert.equal(a.healthy, true);
  assert.match(a.why, /2 deferred messages have waited longer than 30 minutes, only to dead\.example, which reads as the far side's/);
  assert.match(a.why, /2 days old to a@dead\.example, waits because: connect to mx\.dead\.example/);
  assert.deepEqual(a.queue?.overdueDomains, ["dead.example"]);
});

test("overdue mail to three domains is unhealthy, whatever the reason", async () => {
  const timedOut = "connect to mx[192.0.2.1]:25: Connection timed out";
  const queue = [
    deferred("AAA111", 7200, timedOut, "a@one.example"),
    deferred("BBB222", 3600, timedOut, "b@two.example"),
    deferred("CCC333", 3600, timedOut, "c@three.example"),
  ].join("\n");
  const a = await mailFlows(container({ queue }).exec, "mailu-smtp", NOW);
  assert.equal(a.state, "unhealthy");
  assert.match(a.why, /3 of 3 deferred messages have waited longer than 30 minutes, to 3 domains \(one\.example, two\.example, three\.example\)/);
  assert.match(a.why, /the oldest, 2 hours old to a@one\.example/);
});

test("a deferred message younger than the bound is healthy, and is said", async () => {
  const queue = deferred("BBB222", DEFERRED_BOUND_SECONDS - 60, "connect to mx.example.net: Connection timed out");
  const a = await mailFlows(container({ queue }).exec, "mailu-smtp", NOW);
  assert.equal(a.state, "healthy");
  assert.equal(a.healthy, true);
  assert.match(a.why, /1 deferred, the oldest 29 minutes old, within 30 minutes/);
});

test("a container that cannot be reached is unhealthy, and says what docker said", async () => {
  const gone = failed("Error response from daemon: No such container: mailu-smtp\n");
  const a = await mailFlows(container({ status: gone, queue: gone, resolve: gone }).exec, "mailu-smtp", NOW);
  assert.equal(a.state, "unhealthy");
  assert.equal(a.healthy, false);
  assert.match(a.why, /postfix in mailu-smtp does not run: Error response from daemon: No such container: mailu-smtp\./);
  assert.match(a.why, /postfix's queue in mailu-smtp could not be read: Error response from daemon: No such container/);
  assert.equal(a.queue, undefined);
});

test("a name that does not resolve is unhealthy, though the queue is empty", async () => {
  const a = await mailFlows(
    container({ queue: "", resolve: failed("gaierror: [Errno -3] Try again\n") }).exec,
    "mailu-smtp",
    NOW,
  );
  assert.equal(a.state, "unhealthy");
  assert.match(a.why, /mailu-smtp cannot resolve gmail-smtp-in\.l\.google\.com through its resolver: gaierror: \[Errno -3\] Try again\./);
});

test("a container that does not answer in time is said so", async () => {
  const hung = Object.assign(new Error("Command failed"), { killed: true, signal: "SIGTERM" });
  const a = await mailFlows(container({ queue: hung }).exec, "mailu-smtp", NOW);
  assert.equal(a.state, "unhealthy");
  assert.match(a.why, /postfix's queue in mailu-smtp could not be read: it did not answer in time\./);
});

test("the health tool is served without the admin API, on the smtp container it is told", async () => {
  const { collectTools } = await import("@novox/mesh-sdk/tools");
  await import("../dist/tools/index.js");
  const mailu = collectTools({ MESH_MAILU_SMTP_CONTAINER: "mailu-smtp" }).find((m) => m.module === "mailu");
  const names = (mailu?.tools ?? []).map((t) => t.name);
  assert.deepEqual(names, ["mailu_health"]);
  assert.match(mailu!.tools[0].description, /inside mailu-smtp/);
});