// Whether mail flows through mailu, as the node-engine asks it every interval (novox/hq ADR 0240, kind // `tool`; issue 690). Every container read healthy on 2026-10-14 while postfix could not resolve a name: // inbound mail was refused with 450 and outbound mail sat in the deferred queue for days. The parts' // checks judge parts; this judges the flow, from inside the postfix container, the way postfix sees it: // // 1. postfix runs (`postfix status`, standing in for the image's own check, `! /usr/libexec/postfix/master // -t`, which this one replaces); // 2. the deferred queue (`postqueue -j`) shows no fault of this side: a recipient deferred longer than the // bound because a lookup failed here, or overdue mail to three or more domains. One or two far domains // that do not answer are theirs, and can stay deferred for days: said, never unhealthy, so one dead // recipient does not page the operator for days; // 3. a public mail host's name resolves through the resolver postfix is given: a resolver that // SERVFAILs refuses inbound mail before anything reaches the queue. // // The answer is the health answer every health tool gives (mesh-sdk go/health): `state`, `healthy` beside // it for an older reader, and `why` in words — counts, the oldest message's age and its first reason — // never a bare false. import { execFile } from "node:child_process"; import { promisify } from "node:util"; const run = promisify(execFile); /** How long a message may wait in the deferred queue before mail is said not to flow. */ export const DEFERRED_BOUND_SECONDS = 30 * 60; /** The public name resolved through postfix's resolver: gmail's mail exchanger, which mail needs. */ export const PUBLIC_NAME = "gmail-smtp-in.l.google.com"; // The lookup postfix's own would make, through the resolver the container is given; a failure is said on one // line ("gaierror: [Errno -3] Try again" is what a SERVFAIL reads as), never as a traceback. const RESOLVE = [ "import socket, sys", "try:", ` socket.getaddrinfo(${JSON.stringify(PUBLIC_NAME)}, 25)`, "except Exception as e:", " sys.exit(type(e).__name__ + ': ' + str(e))", ].join("\n"); /** Overdue mail to this many recipient domains or more is our side's fault, not one far server's. */ export const SPREAD_DOMAINS = 3; /** * What postfix writes when its own lookups fail, as on 2026-10-14 (issue 687): "Host or domain name not * found. Name service error for name=gmail.com type=MX: Host not found, try again", or "Temporary lookup * failure". A far server that is down reads otherwise ("Connection timed out", "Connection refused"). */ export const LOCAL_LOOKUP_FAILURE = /Name service error|Temporary lookup failure|Host or domain name not found.*try again/is; /** The domain of an address, lower-cased; "(no recipient)" when there is none. */ function domainOf(address?: string): string { const at = address?.lastIndexOf("@") ?? -1; return address && at >= 0 ? address.slice(at + 1).toLowerCase() : "(no recipient)"; } /** Runs a command in a container: its stdout, or a rejection carrying stderr. */ export type Exec = (container: string, command: string[]) => Promise; /** The health answer (mesh-sdk go/health): healthy, or unhealthy with why. */ export interface HealthAnswer { state: "healthy" | "unhealthy"; healthy: boolean; why: string; queue?: { messages: number; deferred: number; overdue: number; oldestDeferredSeconds?: number; overdueDomains?: string[]; nameServiceFailures?: number; }; } /** One message as `postqueue -j` prints it, one JSON object a line (postfix 3.1 and later). */ interface Queued { queue_name?: string; queue_id?: string; arrival_time?: number; recipients?: Array<{ address?: string; delay_reason?: string }>; } /** docker exec, bounded: a hung container is an answer too. */ export function dockerExec(timeoutMs = 10_000): Exec { return async (container, command) => (await run("docker", ["exec", container, ...command], { timeout: timeoutMs })).stdout; } /** What a failed exec said, on one line: its stderr when it has one, else its message. */ function said(e: unknown): string { const err = e as { stderr?: string; message?: string; killed?: boolean; signal?: string }; if (err?.killed || err?.signal === "SIGTERM") return "it did not answer in time"; const text = (err?.stderr ?? "").trim() || (err?.message ?? String(e)).trim(); return text.split("\n").map((l) => l.trim()).filter(Boolean)[0] ?? "it failed and said nothing"; } /** An age in words: "2 days 3 hours", "45 minutes". */ export function age(seconds: number): string { const s = Math.max(0, Math.floor(seconds)); const d = Math.floor(s / 86400), h = Math.floor((s % 86400) / 3600), m = Math.floor((s % 3600) / 60); const unit = (n: number, w: string) => `${n} ${w}${n === 1 ? "" : "s"}`; if (d > 0) return h > 0 ? `${unit(d, "day")} ${unit(h, "hour")}` : unit(d, "day"); if (h > 0) return m > 0 ? `${unit(h, "hour")} ${unit(m, "minute")}` : unit(h, "hour"); return unit(m, "minute"); } function parseQueue(output: string): Queued[] { const queued: Queued[] = []; for (const line of output.split("\n")) { const text = line.trim(); if (!text.startsWith("{")) continue; // "Mail queue is empty" and the like queued.push(JSON.parse(text) as Queued); } return queued; } /** * Judge whether mail flows. `now` is in seconds since the epoch. Every part is asked even when one * fails, so the answer says everything that is wrong at once. */ export async function mailFlows( exec: Exec, container: string, now: number = Date.now() / 1000, boundSeconds: number = DEFERRED_BOUND_SECONDS, ): Promise { const faults: string[] = []; const [status, queue, resolve] = await Promise.allSettled([ exec(container, ["postfix", "status"]), exec(container, ["postqueue", "-j"]), exec(container, ["python3", "-c", RESOLVE]), ]); if (status.status === "rejected") { faults.push(`postfix in ${container} does not run: ${said(status.reason)}.`); } let summary: HealthAnswer["queue"]; const notes: string[] = []; if (queue.status === "rejected") { faults.push(`postfix's queue in ${container} could not be read: ${said(queue.reason)}.`); } else { let queued: Queued[] = []; try { queued = parseQueue(queue.value); } catch (e) { faults.push(`postfix's queue in ${container} printed a line that is not JSON: ${said(e)}.`); } const deferred = queued.filter((q) => q.queue_name === "deferred"); const overdue = deferred .filter((q) => typeof q.arrival_time === "number" && now - q.arrival_time > boundSeconds) .sort((a, b) => (a.arrival_time ?? 0) - (b.arrival_time ?? 0)); const oldest = deferred.reduce( (o, q) => (typeof q.arrival_time === "number" && (o === undefined || q.arrival_time < o) ? q.arrival_time : o), undefined, ); // Each overdue recipient, oldest message first: what our side is judged by, not one far host. const late = overdue.flatMap((q) => (q.recipients?.length ? q.recipients : [{}]).map((r) => ({ to: r.address, domain: domainOf(r.address), reason: r.delay_reason?.trim() || "postfix gave no reason", waited: now - (q.arrival_time ?? now), })), ); const lookups = late.filter((r) => LOCAL_LOOKUP_FAILURE.test(r.reason)); const domains = [...new Set(late.map((r) => r.domain))]; summary = { messages: queued.length, deferred: deferred.length, overdue: overdue.length, ...(oldest !== undefined ? { oldestDeferredSeconds: Math.max(0, Math.floor(now - oldest)) } : {}), ...(domains.length > 0 ? { overdueDomains: domains } : {}), ...(lookups.length > 0 ? { nameServiceFailures: lookups.length } : {}), }; const one = (r: (typeof late)[number]) => `the oldest, ${age(r.waited)} old${r.to ? ` to ${r.to}` : ""}, waits because: ${r.reason}`; const bound = age(boundSeconds); if (lookups.length > 0) { faults.push( `${lookups.length} deferred ${lookups.length === 1 ? "recipient has" : "recipients have"} waited longer ` + `than ${bound} on a name-service failure, which is this server's resolver; ${one(lookups[0])}.`, ); } else if (domains.length >= SPREAD_DOMAINS) { faults.push( `${overdue.length} of ${deferred.length} deferred messages have waited longer than ${bound}, to ` + `${domains.length} domains (${domains.join(", ")}): more than one far server's fault; ${one(late[0])}.`, ); } else if (late.length > 0) { notes.push( `${overdue.length} deferred ${overdue.length === 1 ? "message has" : "messages have"} waited longer than ` + `${bound}, only to ${domains.join(" and ")}, which reads as the far side's: ${one(late[0])}.`, ); } } if (resolve.status === "rejected") { faults.push(`${container} cannot resolve ${PUBLIC_NAME} through its resolver: ${said(resolve.reason)}.`); } if (faults.length > 0) { const why = [`Mail does not flow.`, ...faults, ...notes].join(" "); return { state: "unhealthy", healthy: false, why, queue: summary }; } const waiting = !summary || summary.deferred === 0 ? " The deferred queue is empty." : notes.length > 0 ? ` ${notes.join(" ")}` : ` ${summary.deferred} deferred, the oldest ${age(summary.oldestDeferredSeconds ?? 0)} old, within ${age(boundSeconds)}.`; return { state: "healthy", healthy: true, why: `Mail flows: postfix runs, ${PUBLIC_NAME} resolves.${waiting}`, queue: summary, }; }