Judge mailu by whether mail flows, not only by its parts

On 2026-10-14 every mailu container read healthy while postfix could not
resolve a name: inbound mail was refused with 450 and outbound mail sat in
the deferred queue for days (issue 690). The smtp container is now judged
by a tool check, mailu_health, asked inside it: postfix runs (the image's
own check, kept), no message waits deferred longer than 30 minutes, and a
public mail host resolves through the resolver postfix is given.

modules/mailu/health.ts

@@ -0,0 +1,163 @@
+// Whether mail flows through mailu, as the node-engine asks it every interval (novox/hq ADR 0240, kind
+// `tool`; issue 690). Every container read healthy on 2026-10-14 while postfix could not resolve a name:
+// inbound mail was refused with 450 and outbound mail sat in the deferred queue for days. The parts'
+// checks judge parts; this judges the flow, from inside the postfix container, the way postfix sees it:
+//
+//   1. postfix runs (`postfix status`, the check the image itself ships, which this one replaces);
+//   2. no message waits in the deferred queue longer than the bound (`postqueue -j`): a message that
+//      cannot be delivered for that long is mail that does not flow, whatever the reason;
+//   3. a public mail host's name resolves through the resolver postfix is given: a resolver that
+//      SERVFAILs refuses inbound mail before anything reaches the queue.
+//
+// The answer is the health answer every health tool gives (mesh-sdk go/health): `state`, `healthy` beside
+// it for an older reader, and `why` in words — counts, the oldest message's age and its first reason —
+// never a bare false.
+
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+
+const run = promisify(execFile);
+
+/** How long a message may wait in the deferred queue before mail is said not to flow. */
+export const DEFERRED_BOUND_SECONDS = 30 * 60;
+
+/** The public name resolved through postfix's resolver: gmail's mail exchanger, which mail needs. */
+export const PUBLIC_NAME = "gmail-smtp-in.l.google.com";
+
+// The lookup postfix's own would make, through the resolver the container is given; a failure is said on one
+// line ("gaierror: [Errno -3] Try again" is what a SERVFAIL reads as), never as a traceback.
+const RESOLVE = [
+  "import socket, sys",
+  "try:",
+  `    socket.getaddrinfo(${JSON.stringify(PUBLIC_NAME)}, 25)`,
+  "except Exception as e:",
+  "    sys.exit(type(e).__name__ + ': ' + str(e))",
+].join("\n");
+
+/** Runs a command in a container: its stdout, or a rejection carrying stderr. */
+export type Exec = (container: string, command: string[]) => Promise<string>;
+
+/** The health answer (mesh-sdk go/health): healthy, or unhealthy with why. */
+export interface HealthAnswer {
+  state: "healthy" | "unhealthy";
+  healthy: boolean;
+  why: string;
+  queue?: { messages: number; deferred: number; overdue: number; oldestDeferredSeconds?: number };
+}
+
+/** One message as `postqueue -j` prints it, one JSON object a line (postfix 3.1 and later). */
+interface Queued {
+  queue_name?: string;
+  queue_id?: string;
+  arrival_time?: number;
+  recipients?: Array<{ address?: string; delay_reason?: string }>;
+}
+
+/** docker exec, bounded: a hung container is an answer too. */
+export function dockerExec(timeoutMs = 10_000): Exec {
+  return async (container, command) =>
+    (await run("docker", ["exec", container, ...command], { timeout: timeoutMs })).stdout;
+}
+
+/** What a failed exec said, on one line: its stderr when it has one, else its message. */
+function said(e: unknown): string {
+  const err = e as { stderr?: string; message?: string; killed?: boolean; signal?: string };
+  if (err?.killed || err?.signal === "SIGTERM") return "it did not answer in time";
+  const text = (err?.stderr ?? "").trim() || (err?.message ?? String(e)).trim();
+  return text.split("\n").map((l) => l.trim()).filter(Boolean)[0] ?? "it failed and said nothing";
+}
+
+/** An age in words: "2 days 3 hours", "45 minutes". */
+export function age(seconds: number): string {
+  const s = Math.max(0, Math.floor(seconds));
+  const d = Math.floor(s / 86400), h = Math.floor((s % 86400) / 3600), m = Math.floor((s % 3600) / 60);
+  const unit = (n: number, w: string) => `${n} ${w}${n === 1 ? "" : "s"}`;
+  if (d > 0) return h > 0 ? `${unit(d, "day")} ${unit(h, "hour")}` : unit(d, "day");
+  if (h > 0) return m > 0 ? `${unit(h, "hour")} ${unit(m, "minute")}` : unit(h, "hour");
+  return unit(m, "minute");
+}
+
+function parseQueue(output: string): Queued[] {
+  const queued: Queued[] = [];
+  for (const line of output.split("\n")) {
+    const text = line.trim();
+    if (!text.startsWith("{")) continue; // "Mail queue is empty" and the like
+    queued.push(JSON.parse(text) as Queued);
+  }
+  return queued;
+}
+
+/**
+ * Judge whether mail flows. `now` is in seconds since the epoch. Every part is asked even when one
+ * fails, so the answer says everything that is wrong at once.
+ */
+export async function mailFlows(
+  exec: Exec,
+  container: string,
+  now: number = Date.now() / 1000,
+  boundSeconds: number = DEFERRED_BOUND_SECONDS,
+): Promise<HealthAnswer> {
+  const faults: string[] = [];
+  const [status, queue, resolve] = await Promise.allSettled([
+    exec(container, ["postfix", "status"]),
+    exec(container, ["postqueue", "-j"]),
+    exec(container, ["python3", "-c", RESOLVE]),
+  ]);
+
+  if (status.status === "rejected") {
+    faults.push(`postfix in ${container} does not run: ${said(status.reason)}.`);
+  }
+
+  let summary: HealthAnswer["queue"];
+  if (queue.status === "rejected") {
+    faults.push(`postfix's queue in ${container} could not be read: ${said(queue.reason)}.`);
+  } else {
+    let queued: Queued[] = [];
+    try {
+      queued = parseQueue(queue.value);
+    } catch (e) {
+      faults.push(`postfix's queue in ${container} printed a line that is not JSON: ${said(e)}.`);
+    }
+    const deferred = queued.filter((q) => q.queue_name === "deferred");
+    const overdue = deferred
+      .filter((q) => typeof q.arrival_time === "number" && now - q.arrival_time > boundSeconds)
+      .sort((a, b) => (a.arrival_time ?? 0) - (b.arrival_time ?? 0));
+    const oldest = deferred.reduce<number | undefined>(
+      (o, q) => (typeof q.arrival_time === "number" && (o === undefined || q.arrival_time < o) ? q.arrival_time : o),
+      undefined,
+    );
+    summary = {
+      messages: queued.length,
+      deferred: deferred.length,
+      overdue: overdue.length,
+      ...(oldest !== undefined ? { oldestDeferredSeconds: Math.max(0, Math.floor(now - oldest)) } : {}),
+    };
+    if (overdue.length > 0) {
+      const first = overdue[0];
+      const reason = first.recipients?.find((r) => r.delay_reason)?.delay_reason?.trim();
+      const to = first.recipients?.[0]?.address;
+      faults.push(
+        `${overdue.length} of ${deferred.length} deferred messages have waited longer than ${age(boundSeconds)}; ` +
+          `the oldest, ${age(now - (first.arrival_time ?? now))} old${to ? ` to ${to}` : ""}, ` +
+          `waits because: ${reason || "postfix gave no reason"}.`,
+      );
+    }
+  }
+
+  if (resolve.status === "rejected") {
+    faults.push(`${container} cannot resolve ${PUBLIC_NAME} through its resolver: ${said(resolve.reason)}.`);
+  }
+
+  if (faults.length > 0) {
+    return { state: "unhealthy", healthy: false, why: `Mail does not flow. ${faults.join(" ")}`, queue: summary };
+  }
+  const waiting = summary && summary.deferred > 0
+    ? ` ${summary.deferred} deferred, the oldest ${age(summary.oldestDeferredSeconds ?? 0)} old, within ${age(boundSeconds)}.`
+    : " The deferred queue is empty.";
+  return {
+    state: "healthy",
+    healthy: true,
+    why: `Mail flows: postfix runs, ${PUBLIC_NAME} resolves.${waiting}`,
+    queue: summary,
+  };
+}

modules/mailu/module.json

@@ -443,7 +443,12 @@
       "name": "mailu-smtp",
       "image": "ghcr.io/mailu/postfix@sha256:e2e49f39e53b80eac9e7a2f18d9df11edeb4914fd62dbba89b3155e8e034f62e",
       "health": {
-        "kind": "runtime"
+        "kind": "tool",
+        "tool": "mailu_health",
+        "interval": "60s",
+        "timeout": "30s",
+        "looks": 3,
+        "grace": "120s"
       },
       "network": "mailu",
       "env-file": [
@@ -642,6 +647,7 @@
           "MESH_MAILU_URL": "http://127.0.0.1:${port:8080}/api/v1",
           "MESH_MAILU_API_KEY_FILE": "${dir:state}/api-token.secret",
           "MESH_MAILU_IMAP_CONTAINER": "mailu-imap",
+          "MESH_MAILU_SMTP_CONTAINER": "mailu-smtp",
           "MESH_MAILU_CONFIG_FILE": "${dir:mesh-state}/config.json",
           "MESH_RECEIVES": "${dir:grants}/mesh.json"
         }

modules/mailu/test/mail-flows.test.ts

@@ -0,0 +1,111 @@
+// Whether mail flows (novox/hq issue 690): the health tool is unhealthy, in words, when a message waits in
+// postfix's deferred queue longer than the bound, when postfix does not run, when a public name does not
+// resolve, and when the container cannot be reached — and healthy otherwise. Run against the compiled
+// module, the way the runtime loads it.
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+
+import { mailFlows, DEFERRED_BOUND_SECONDS } from "../dist/health.js";
+
+const NOW = 1_800_000_000;
+
+type Answers = { status?: string | Error; queue?: string | Error; resolve?: string | Error };
+
+/** A container that answers each command as told; an Error is a failed exec, with its stderr. */
+function container(a: Answers) {
+  const asked: string[][] = [];
+  const exec = async (_name: string, command: string[]) => {
+    asked.push(command);
+    const answer = command[0] === "postfix" ? a.status ?? "" : command[0] === "postqueue" ? a.queue ?? "" : a.resolve ?? "";
+    if (answer instanceof Error) throw answer;
+    return answer;
+  };
+  return { exec, asked };
+}
+
+function failed(stderr: string): Error {
+  return Object.assign(new Error("Command failed: docker exec"), { stderr });
+}
+
+function deferred(id: string, ageSeconds: number, reason: string): string {
+  return JSON.stringify({
+    queue_name: "deferred",
+    queue_id: id,
+    arrival_time: NOW - ageSeconds,
+    message_size: 1234,
+    sender: "someone@example.org",
+    recipients: [{ address: "friend@example.com", delay_reason: reason }],
+  });
+}
+
+test("an empty queue is healthy, and says so", async () => {
+  const c = container({ queue: "" });
+  const a = await mailFlows(c.exec, "mailu-smtp", NOW);
+  assert.equal(a.state, "healthy");
+  assert.equal(a.healthy, true);
+  assert.match(a.why, /deferred queue is empty/);
+  assert.deepEqual(a.queue, { messages: 0, deferred: 0, overdue: 0 });
+  assert.equal(c.asked.length, 3, "postfix, its queue and a lookup were each asked");
+});
+
+test("a deferred message older than the bound is unhealthy, with counts, its age and its reason", async () => {
+  const reason = "Host or domain name not found. Name service error for name=example.com type=MX: Host not found, try again";
+  const queue = [
+    deferred("AAA111", 3 * 86400 + 2 * 3600, reason),
+    deferred("BBB222", 5 * 60, "connect to mx.example.net[192.0.2.1]:25: Connection timed out"),
+    JSON.stringify({ queue_name: "active", queue_id: "CCC333", arrival_time: NOW - 10, recipients: [] }),
+  ].join("\n");
+  const a = await mailFlows(container({ queue }).exec, "mailu-smtp", NOW);
+  assert.equal(a.state, "unhealthy");
+  assert.equal(a.healthy, false);
+  assert.match(a.why, /^Mail does not flow\./);
+  assert.match(a.why, /1 of 2 deferred messages have waited longer than 30 minutes/);
+  assert.match(a.why, /3 days 2 hours old to friend@example\.com/);
+  assert.ok(a.why.includes(reason), "the first reason is said");
+  assert.deepEqual(a.queue, { messages: 3, deferred: 2, overdue: 1, oldestDeferredSeconds: 3 * 86400 + 2 * 3600 });
+});
+
+test("a deferred message younger than the bound is healthy, and is said", async () => {
+  const queue = deferred("BBB222", DEFERRED_BOUND_SECONDS - 60, "connect to mx.example.net: Connection timed out");
+  const a = await mailFlows(container({ queue }).exec, "mailu-smtp", NOW);
+  assert.equal(a.state, "healthy");
+  assert.equal(a.healthy, true);
+  assert.match(a.why, /1 deferred, the oldest 29 minutes old, within 30 minutes/);
+});
+
+test("a container that cannot be reached is unhealthy, and says what docker said", async () => {
+  const gone = failed("Error response from daemon: No such container: mailu-smtp\n");
+  const a = await mailFlows(container({ status: gone, queue: gone, resolve: gone }).exec, "mailu-smtp", NOW);
+  assert.equal(a.state, "unhealthy");
+  assert.equal(a.healthy, false);
+  assert.match(a.why, /postfix in mailu-smtp does not run: Error response from daemon: No such container: mailu-smtp\./);
+  assert.match(a.why, /postfix's queue in mailu-smtp could not be read: Error response from daemon: No such container/);
+  assert.equal(a.queue, undefined);
+});
+
+test("a name that does not resolve is unhealthy, though the queue is empty", async () => {
+  const a = await mailFlows(
+    container({ queue: "", resolve: failed("gaierror: [Errno -3] Try again\n") }).exec,
+    "mailu-smtp",
+    NOW,
+  );
+  assert.equal(a.state, "unhealthy");
+  assert.match(a.why, /mailu-smtp cannot resolve gmail-smtp-in\.l\.google\.com through its resolver: gaierror: \[Errno -3\] Try again\./);
+});
+
+test("a container that does not answer in time is said so", async () => {
+  const hung = Object.assign(new Error("Command failed"), { killed: true, signal: "SIGTERM" });
+  const a = await mailFlows(container({ queue: hung }).exec, "mailu-smtp", NOW);
+  assert.equal(a.state, "unhealthy");
+  assert.match(a.why, /postfix's queue in mailu-smtp could not be read: it did not answer in time\./);
+});
+
+test("the health tool is served without the admin API, on the smtp container it is told", async () => {
+  const { collectTools } = await import("@novox/mesh-sdk/tools");
+  await import("../dist/tools/index.js");
+  const mailu = collectTools({ MESH_MAILU_SMTP_CONTAINER: "mailu-smtp" }).find((m) => m.module === "mailu");
+  const names = (mailu?.tools ?? []).map((t) => t.name);
+  assert.deepEqual(names, ["mailu_health"]);
+  assert.match(mailu!.tools[0].description, /inside mailu-smtp/);
+});

modules/mailu/tools/index.ts

@@ -4,6 +4,26 @@
 
 import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools";
 import { MailuClient } from "../client.js";
+import { dockerExec, mailFlows, DEFERRED_BOUND_SECONDS, PUBLIC_NAME } from "../health.js";
+
+/**
+ * Whether mail flows (novox/hq ADR 0240, issue 690): the tool the node-engine asks as the smtp container's
+ * check. It needs no admin API, only the postfix container, so it is served even when the API is not
+ * configured — a check that vanished with the API would say unhealthy for the wrong reason.
+ */
+export function getHealthTool(env: NodeJS.ProcessEnv): ToolDefinition {
+  const container = env.MESH_MAILU_SMTP_CONTAINER || "mailu-smtp";
+  return {
+    name: "mailu_health",
+    description:
+      `Whether mail flows through mailu, asked inside ${container}: postfix runs, no message has waited in ` +
+      `its deferred queue longer than ${DEFERRED_BOUND_SECONDS / 60} minutes, and ${PUBLIC_NAME} resolves ` +
+      "through the resolver postfix uses. Unhealthy with counts, the oldest message's age and its first reason " +
+      "(novox/hq ADR 0240, issue 690).",
+    input: {},
+    run: async () => mailFlows(dockerExec(), container),
+  };
+}
 
 export function getMailuTools(mailu: MailuClient): ToolDefinition[] {
   return [
@@ -143,12 +163,14 @@ export function getMailuTools(mailu: MailuClient): ToolDefinition[] {
   ];
 }
 
-// The tools exist only when the admin API is configured; without it, mailu contributes none rather
-// than failing the whole runtime.
+// The admin tools exist only when the admin API is configured; without it, mailu contributes only its
+// health tool rather than failing the whole runtime.
 registerModuleTools("mailu", (env) => {
+  let tools: ToolDefinition[] = [];
   try {
-    return getMailuTools(MailuClient.fromEnv(env));
+    tools = getMailuTools(MailuClient.fromEnv(env));
   } catch {
-    return [];
+    tools = [];
   }
+  return [...tools, getHealthTool(env)];
 });

modules/mailu/tsconfig.json

@@ -8,5 +8,5 @@
     "skipLibCheck": true,
     "noEmit": true
   },
-  "include": ["client.ts", "index.ts", "tools/index.ts"]
+  "include": ["client.ts", "health.ts", "index.ts", "tools/index.ts"]
 }