Merge remote-tracking branch 'origin/feat/r4d-automx-go' into feat/the-checker

This commit is contained in:
2026-10-21 21:50:26 +02:00
24 changed files with 846 additions and 495 deletions
+1 -1
View File
@@ -1,4 +1,4 @@
59
58
The long-running resources of this catalogue that do not say how they are ready (`health`, novox/hq ADR 0240
rule 8), as `module check` counts them. It may only go down: merge-check.sh fails a change that raises it, and
one that lowers it writes the new number on the first line. From 2026-11-18, or once it is 0, a long-running
+13 -1
View File
@@ -16,6 +16,18 @@ installation's are settings on the assignment, `settings set mailu <file>`:
header from — the address the proxy forwards with. The front's own hostname is the name of its
`web` route, told to it by the mesh.
## Mail client autoconfiguration
`mailu-automx` answers what mail clients ask for their settings (Thunderbird, Outlook, Apple) for the domains in
`AMX_MAIL_DOMAINS`, naming the IMAP and SMTP servers and ports of mailu's environment (`AMX_IMAP_ADDRESS`,
`AMX_IMAP_PORT`, `AMX_SMTP_ADDRESS`, `AMX_SMTP_PORT`). It is `cmd/automx`, a Go program of the standard library
only in this module's `go.mod` (ADR 0480 clauses 1, 2, 9). It runs in the mirrored alpine image the route proxy's
`trust` step runs, from the `program-automx` archive mounted read-only at `/opt/mesh`; the container is restarted
with the archive and with `mailu.env`, which the program reads at each start. It keeps no database. The file the
Python automx kept in `data-automx` is left where it is on the machine, not deleted by the mesh (ADR 0480
clause 10). The answers it gave before are the fixtures in `cmd/automx/testdata/captured`, with how they were
taken (`CAPTURED`) and what the program answers differently (`DIFFERENCES`).
## Files it does not know
Each directory or archive this module declares says how it treats a file it did not write (ADR 0422, ADR 0416).
@@ -23,7 +35,7 @@ Each directory or archive this module declares says how it treats a file it did
- `${dir:mesh-state}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
- `${dir:state}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
- `${dir:grants}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
- `${dir:data-automx}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
- the archive `program-automx` (`${dir:state}/automx`): everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
- `${dir:data-certs}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
- `${dir:data-data}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
- `${dir:data-dkim}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
-87
View File
@@ -1,87 +0,0 @@
// What the autoconfiguration module (automx2) holds, as stored: its provider, server, domain and server_domain
// rows, read from its SQLite file inside its container with the sqlite3 program the image carries, opened
// read-only. Nothing is written and nothing is interpreted: the rows are the evidence for what automx answers
// to mail clients (novox/hq issue 1630, ADR 0480), and the reason the answer may be empty.
import type { Exec } from "./health.js";
export const AUTOMX_DATABASE = "/data/db.sqlite";
/** The tables read, each with its columns and its ordering. Fixed text: the tool takes no query from its caller. */
export const AUTOMX_TABLES: Record<string, { columns: string[]; order: string }> = {
provider: { columns: ["id", "name", "short_name"], order: "id" },
server: { columns: ["id", "prio", "name", "port", "type", "socket_type", "user_name", "authentication"], order: "id" },
domain: { columns: ["id", "name", "provider_id", "ldapserver_id"], order: "id" },
server_domain: { columns: ["server_id", "domain_id"], order: "domain_id, server_id" },
};
/**
* The one statement that reads a table. The sqlite3 in the mailu-automx image does not know `-json` (it answers
* "sqlite3: Error: unknown option: -json", issue 1630), so each column is printed as `<type>:<hex of its bytes>`, which
* the oldest sqlite3 prints in its default list mode: no quoting, no separator or newline inside a cell, and a NULL
* (`null:`) apart from an empty text (`text:`).
*/
export function selectOf(table: string): string {
const t = AUTOMX_TABLES[table];
return `SELECT ${t.columns.map((c) => `typeof(${c}) || ':' || hex(${c})`).join(", ")} FROM ${table} ORDER BY ${t.order}`;
}
export type Row = Record<string, string | number | null>;
export type AutomxRows = {
database: string;
tables: Record<string, Row[]>;
/** servers whose name is empty: the client is told an empty host name for them */
empty_names: { id: number; type: string }[];
/** tables that could not be read, each with what sqlite3 or docker said on one line */
unreadable: Record<string, string>;
};
/** One cell as selectOf prints it: `<type>:<hex>`; text is the bytes read as UTF-8, a number is read from its text. */
function cellOf(cell: string): string | number | null {
const at = cell.indexOf(":");
if (at < 0 || cell.length - at - 1 > 0 && !/^([0-9A-Fa-f]{2})+$/.test(cell.slice(at + 1))) throw new Error(`sqlite3 printed ${JSON.stringify(cell.slice(0, 40))}, not a cell`);
const type = cell.slice(0, at);
const bytes = Buffer.from(cell.slice(at + 1), "hex").toString("utf8");
switch (type) {
case "null": return null;
case "integer": case "real": return Number(bytes);
default: return bytes;
}
}
/** sqlite3 prints one line per row, cells apart by `|`, and nothing at all for a table without rows. */
export function parseRows(table: string, output: string): Row[] {
const columns = AUTOMX_TABLES[table].columns;
const rows: Row[] = [];
for (const line of output.split("\n")) {
if (line.trim() === "") continue;
const cells = line.replace(/\r$/, "").split("|");
if (cells.length !== columns.length) throw new Error(`sqlite3 printed ${cells.length} cells for ${columns.length} columns`);
rows.push(Object.fromEntries(columns.map((c, i) => [c, cellOf(cells[i])])));
}
return rows;
}
/** The command that reads one table: read-only, one fixed statement. */
export function readCommand(table: string): string[] {
return ["sqlite3", "-readonly", AUTOMX_DATABASE, selectOf(table)];
}
/** Read every table; one that fails is said in `unreadable` and does not hide the others. */
export async function readAutomx(exec: Exec, container: string): Promise<AutomxRows> {
const answer: AutomxRows = { database: AUTOMX_DATABASE, tables: {}, empty_names: [], unreadable: {} };
for (const table of Object.keys(AUTOMX_TABLES)) {
try {
answer.tables[table] = parseRows(table, await exec(container, readCommand(table)));
} catch (e) {
const err = e as { stderr?: string; message?: string };
const text = (err?.stderr ?? "").trim() || (err?.message ?? String(e)).trim();
answer.unreadable[table] = text.split("\n").map((l) => l.trim()).filter(Boolean)[0] ?? "it failed and said nothing";
}
}
for (const s of answer.tables.server ?? []) {
if (String(s.name ?? "").trim() === "") answer.empty_names.push({ id: Number(s.id), type: String(s.type) });
}
return answer;
}
-44
View File
@@ -1,44 +0,0 @@
# automx2 — the autoconfig/autodiscover responder, carried by the mailu module as its own
# artifact: it is a config-baked sidecar of this mail server, not a standalone application
# (novox/hq ADR 0015 draws that line at applications).
#
# The base is named rather than pinned (novox/hq issue 044): declared in module.json's
# `build.on`. The build context is the module's own directory; every ADD says so.
ARG PYTHON_BASE
FROM ${PYTHON_BASE}
RUN apk add --no-cache bash sqlite
WORKDIR /automx2
ADD automx/files/setupvenv.sh /automx2/setupvenv.sh
ADD automx/files/start /automx2/start
ADD automx/files/setup /automx2/setup
ADD automx/files/setup-db /automx2/setup-db
ADD automx/files/add-domains /automx2/add-domains
RUN chmod u+x setupvenv.sh start add-domains setup setup-db
RUN ./setupvenv.sh \
&& . .venv/bin/activate \
&& pip install automx2==2021.6
# The launcher `start` expects. In the predecessor's image this wrapper appeared during a build
# step that never made it into the files this module carries — the image worked and the recipe
# could not reproduce it. Written here explicitly, verbatim from the proven image, so the build
# is the whole truth about the image again.
RUN mkdir -p .venv/scripts && printf '%s\n' \
'#!/usr/bin/env bash' \
'set -euo pipefail' \
'. .venv/bin/activate' \
"export FLASK_ENV='production'" \
"export FLASK_APP='automx2.server:app'" \
'flask "$@"' > .venv/scripts/flask.sh && chmod +x .venv/scripts/flask.sh
ENV AUTOMX2_CONF=/etc/automx2.conf
ADD automx/files/automx2.conf /etc/automx2.conf
# VOLUME deliberately absent: the anonymous /data volume is exactly what lost db.sqlite on
# every recreate (measured on novox 2026-08-10). The manifest binds a real directory instead.
ENTRYPOINT ["/bin/sh"]
CMD ["./start"]
EXPOSE 4243
-49
View File
@@ -1,49 +0,0 @@
#!/usr/bin/env bash
set -e
echo "${MAIL_DOMAINS}"
# Split domains into array
IFS=', ' read -r -a array <<< "${AMX_MAIL_DOMAINS}"
# User configurable section -- START
PROVIDER_ID=001
SQL_CMD="";
# Iterate domains resulting from split on second arg
for element in "${array[@]}"
do
# Set vars
DOMAIN=$element
PROVIDER_NAME=$DOMAIN
PROVIDER_SHORTNAME=$DOMAIN
# Optional LDAP server
#LDAP_SERVER="ldap.${DOMAIN}"
# User configurable section -- END
s1_id=$((PROVIDER_ID + 1))
s2_id=$((PROVIDER_ID + 2))
s3_id=$((PROVIDER_ID + 3))
dom_id=$((PROVIDER_ID + 4))
s3_id='NULL'
SQL_CMD=$(cat <<EOT
$SQL_CMD
INSERT INTO provider(id, name, short_name) VALUES(${PROVIDER_ID}, '${PROVIDER_NAME}', '${PROVIDER_SHORTNAME}');
INSERT INTO server(id, port, type, name, socket_type, user_name, authentication)
VALUES(${s1_id}, ${AMX_IMAP_PORT}, 'imap', '${AMX_IMAP_SERVER}', 'STARTTLS', '%EMAILADDRESS%', 'password-cleartext');
INSERT INTO server(id, port, type, name, socket_type, user_name, authentication)
VALUES(${s2_id}, ${AMX_SMTP_PORT}, 'smtp', '${AMX_SMTP_ADDRESS}', 'STARTTLS', '%EMAILADDRESS%', 'password-cleartext');
INSERT INTO domain(id, name, provider_id, ldapserver_id) VALUES(${dom_id}, '${DOMAIN}', ${PROVIDER_ID}, ${s3_id});
INSERT INTO server_domain(server_id, domain_id) VALUES(${s1_id}, ${dom_id});
INSERT INTO server_domain(server_id, domain_id) VALUES(${s2_id}, ${dom_id});
EOT
)
PROVIDER_ID=$((PROVIDER_ID+10))
done
echo -e ${SQL_CMD}
echo -e ${SQL_CMD} | sqlite3 /data/db.sqlite
-21
View File
@@ -1,21 +0,0 @@
[automx2]
# A typical production setup would use loglevel = WARNING
loglevel = WARNING
# Echo SQL commands into log? Used for debugging.
db_echo = false
# In-memory SQLite database
# db_uri = sqlite:///:memory:
# SQLite database in a UNIX-like file system
db_uri = sqlite:////data/db.sqlite
# MySQL database on a remote server. This example does not use an encrypted
# connection and is therefore *not* recommended for production use.
#db_uri = mysql://username:password@server.example.com/db
# Number of proxy servers between automx2 and the client (default: 0).
# If your logs only show 127.0.0.1 or ::1 as the source IP for incoming
# connections, proxy_count probably needs to be changed.
proxy_count = 1
-12
View File
@@ -1,12 +0,0 @@
#!/usr/bin/env bash
set -e
if [ ! -e /data/db.sqlite ]; then
# DB SETUP
echo "SETTING UP DB"
./setup-db
echo "ADDING DOMAINS"
# Add the domains
./add-domains
fi
-84
View File
@@ -1,84 +0,0 @@
#!/usr/bin/env bash
set -e
# LDAP-Server
LDAP=$(cat <<EOT
CREATE TABLE ldapserver(
id INT PRIMARY KEY NOT NULL,
name TEXT NOT NULL,
port INT NOT NULL,
use_ssl INT NOT NULL,
search_base TEXT NOT NULL,
search_filter TEXT NOT NULL,
attr_uid TEXT NOT NULL,
attr_cn TEXT NOT NULL,
bind_password TEXT NOT NULL,
bind_user TEXT NOT NULL
);
EOT
)
# Provider
PROVIDER=$(cat <<EOT
CREATE TABLE provider(
id INT PRIMARY KEY NOT NULL,
name TEXT NOT NULL,
short_name TEXT NOT NULL
);
EOT
)
# Server
SERVER=$(cat <<EOT
CREATE TABLE server(
id INT PRIMARY KEY NOT NULL,
prio INT NOT NULL DEFAULT 10,
name TEXT NOT NULL,
port INT NOT NULL,
type TEXT NOT NULL,
socket_type TEXT NOT NULL,
user_name TEXT NOT NULL,
authentication TEXT NOT NULL
);
EOT
)
# Domain
DOMAIN=$(cat <<EOT
CREATE TABLE domain(
id INT PRIMARY KEY NOT NULL,
name TEXT NOT NULL,
provider_id INT NOT NULL,
ldapserver_id INT NULL,
FOREIGN KEY(ldapserver_id) REFERENCES ldapserver(id),
FOREIGN KEY(provider_id) REFERENCES provider(id)
);
CREATE UNIQUE INDEX domain_name ON domain(name);
EOT
)
# Server-Domain
SERVER_DOMAIN=$(cat <<EOT
CREATE TABLE server_domain(
server_id INT NOT NULL,
domain_id INT NOT NULL,
FOREIGN KEY(server_id) REFERENCES server(id),
FOREIGN KEY(domain_id) REFERENCES domain(id)
);
EOT
)
## TODO Foreign keys
SQL_CMD=$(cat <<EOT
$LDAP
$PROVIDER
$SERVER
$DOMAIN
$SERVER_DOMAIN
EOT
)
echo -e ${SQL_CMD}
echo -e ${SQL_CMD} | sqlite3 /data/db.sqlite
-38
View File
@@ -1,38 +0,0 @@
#!/usr/bin/env bash
# vim:ts=4:sw=4:noet
#
# Creates a Python 3 virtual environment. The target directory can be passed
# as a parameter. The default path is '.venv' in the current directory.
dir="${1:-.venv}"
echo "Setup dir $dir"
set -e
if [ -d "${dir}" ]; then
echo >&2 "Directory '${dir}' already exists, exiting."
exit 1
fi
python3 -m venv "${dir}"
source "${dir}/bin/activate"
set +e
pip install -U pip setuptools wheel || true
#set -e
## vim:tabstop=4:noexpandtab
##
## Creates a Python 3 virtual environment. The target directory can be passed
## as a parameter. The default path is 'venv' in the current directory.
#
#dir="${1:-venv}"
#
#set -e
#if [ -d "${dir}" ]; then
# echo "Directory '${dir}' already exists, exiting." >&2
# exit 1
#fi
#python3 -m venv "${dir}"
#. "${dir}/bin/activate"
#
#set +e
#pip install -U pip setuptools || true
-8
View File
@@ -1,8 +0,0 @@
#!/usr/bin/env bash
set -e
# Setup
./setup
# Start
./.venv/scripts/flask.sh run --host=0.0.0.0 --port=4243
+260
View File
@@ -0,0 +1,260 @@
package main
import (
"bytes"
"io"
"net/http"
"net/http/httptest"
"net/url"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"testing"
)
// The environment mailu gives automx on novox when the fixtures were captured (files/mailu.env): the mail domain,
// the route name for both servers, the ports.
var capturedEnvironment = map[string]string{
"AMX_MAIL_DOMAINS": "novox.be",
"AMX_IMAP_ADDRESS": "mail.novox.be",
"AMX_IMAP_PORT": "143",
"AMX_SMTP_ADDRESS": "mail.novox.be",
"AMX_SMTP_PORT": "587",
"AUTOMX_PORT": "4243",
}
func getenvOf(m map[string]string) func(string) string {
return func(k string) string { return m[k] }
}
func testServer(t *testing.T) *httptest.Server {
t.Helper()
cfg, err := configFrom(getenvOf(capturedEnvironment))
if err != nil {
t.Fatal(err)
}
s := httptest.NewServer(newHandler(cfg))
t.Cleanup(s.Close)
return s
}
const captured = "testdata/captured"
// A captured request: its first line is the method and the URL the client used, then the headers sent, then after
// a blank line the body sent (CAPTURED, how the files were written).
type request struct {
method, url string
header [][2]string
body string
}
func readRequest(t *testing.T, stem string) request {
t.Helper()
raw, err := os.ReadFile(stem + ".request")
if err != nil {
t.Fatal(err)
}
head, body, hasBody := strings.Cut(string(raw), "\n\n")
if !hasBody {
head = strings.TrimSuffix(head, "\n")
}
lines := strings.Split(head, "\n")
method, target, ok := strings.Cut(lines[0], " ")
if !ok {
t.Fatalf("%s: first line %q is not METHOD URL", stem, lines[0])
}
r := request{method: method, url: target, body: body}
for _, l := range lines[1:] {
k, v, ok := strings.Cut(l, ": ")
if !ok {
t.Fatalf("%s: header line %q", stem, l)
}
r.header = append(r.header, [2]string{k, v})
}
return r
}
func readFile(t *testing.T, name string) string {
t.Helper()
b, err := os.ReadFile(name)
if err != nil {
t.Fatal(err)
}
return string(b)
}
// A fixture asks for its body to differ only where DIFFERENCES says so: a stem with a .corrected.body is answered
// with that, and its Content-Length is the corrected body's.
func expectedBody(t *testing.T, stem string) (body string, corrected bool) {
if _, err := os.Stat(stem + ".corrected.body"); err == nil {
return readFile(t, stem+".corrected.body"), true
}
return readFile(t, stem+".body"), false
}
func TestEveryCapturedRequestIsAnsweredAsAutomxAnswered(t *testing.T) {
stems, _ := filepath.Glob(filepath.Join(captured, "*.request"))
if len(stems) < 17 {
t.Fatalf("%d captured requests found, want the 17 of issue 1630", len(stems))
}
srv := testServer(t)
client := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
for _, f := range stems {
stem := strings.TrimSuffix(f, ".request")
t.Run(filepath.Base(stem), func(t *testing.T) {
rq := readRequest(t, stem)
u, err := url.Parse(rq.url)
if err != nil {
t.Fatal(err)
}
// The mesh's route reaches the program with the client's name and scheme; the program is told them in
// the forwarding headers (the Location of fixture 14 holds them).
req, err := http.NewRequest(rq.method, srv.URL+u.RequestURI(), strings.NewReader(rq.body))
if err != nil {
t.Fatal(err)
}
for _, h := range rq.header {
req.Header.Set(h[0], h[1])
}
req.Host = u.Host
req.Header.Set("X-Forwarded-Proto", u.Scheme)
resp, err := client.Do(req)
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
got, _ := io.ReadAll(resp.Body)
if want := strings.TrimRight(readFile(t, stem+".status"), "\n"); resp.Status != want {
t.Errorf("status %q, captured %q", resp.Status, want)
}
wantBody, corrected := expectedBody(t, stem)
if !bytes.Equal(got, []byte(wantBody)) {
t.Errorf("body\n%q\nwant\n%q", got, wantBody)
}
want := map[string]string{}
for _, l := range strings.Split(strings.TrimRight(readFile(t, stem+".headers"), "\n"), "\n") {
if k, v, ok := strings.Cut(l, ": "); ok {
want[k] = v
}
}
if corrected {
want["Content-Length"] = strconv.Itoa(len(wantBody))
}
// Server names the Python program that answered (DIFFERENCES); Date changes on every request.
delete(want, "Server")
var extra []string
for k := range resp.Header {
if k == "Date" || k == "Server" {
continue
}
if _, ok := want[k]; !ok {
extra = append(extra, k)
}
}
sort.Strings(extra)
if len(extra) > 0 {
t.Errorf("headers the capture does not have: %v", extra)
}
for k, v := range want {
if g := resp.Header.Get(k); g != v {
t.Errorf("header %s: %q, captured %q", k, g, v)
}
}
})
}
}
// DIFFERENCES: the one change clause 9 of ADR 0480 allows to an answer is the IMAP server's name, written from
// AMX_IMAP_ADDRESS where automx wrote an empty one.
func TestTheOnlyCorrectedAnswerIsTheImapServersName(t *testing.T) {
corrected, _ := filepath.Glob(filepath.Join(captured, "*.corrected.body"))
if len(corrected) != 3 {
t.Fatalf("%d corrected answers, want the three Thunderbird answers (01, 02, 03): %v", len(corrected), corrected)
}
for _, c := range corrected {
orig := readFile(t, strings.TrimSuffix(c, ".corrected.body")+".body")
want := strings.Replace(orig, "<hostname />", "<hostname>mail.novox.be</hostname>", 1)
if strings.Contains(want, "<hostname />") || want == orig {
t.Fatalf("%s: the capture has no single empty hostname to correct", c)
}
if got := readFile(t, c); got != want {
t.Errorf("%s differs from the capture by more than the IMAP server's name", c)
}
}
}
func TestTheImapServerIsNamedFromTheImapAddress(t *testing.T) {
env := map[string]string{}
for k, v := range capturedEnvironment {
env[k] = v
}
env["AMX_IMAP_ADDRESS"] = "imap.example.test"
env["AMX_IMAP_PORT"] = "993"
cfg, err := configFrom(getenvOf(env))
if err != nil {
t.Fatal(err)
}
rec := httptest.NewRecorder()
newHandler(cfg).ServeHTTP(rec, httptest.NewRequest("GET", "/mail/config-v1.1.xml?emailaddress=a@novox.be", nil))
body := rec.Body.String()
if !strings.Contains(body, `<incomingServer type="imap"><hostname>imap.example.test</hostname><port>993</port>`) {
t.Errorf("incoming server not named from AMX_IMAP_ADDRESS and AMX_IMAP_PORT: %s", body)
}
if !strings.Contains(body, `<outgoingServer type="smtp"><hostname>mail.novox.be</hostname><port>587</port>`) {
t.Errorf("outgoing server changed: %s", body)
}
}
func TestEveryDomainOfTheEnvironmentIsServed(t *testing.T) {
env := map[string]string{}
for k, v := range capturedEnvironment {
env[k] = v
}
env["AMX_MAIL_DOMAINS"] = "novox.be, example.test"
cfg, err := configFrom(getenvOf(env))
if err != nil {
t.Fatal(err)
}
h := newHandler(cfg)
for _, d := range []string{"novox.be", "example.test"} {
rec := httptest.NewRecorder()
h.ServeHTTP(rec, httptest.NewRequest("GET", "/mail/config-v1.1.xml?emailaddress=a@"+d, nil))
if rec.Code != 200 || !strings.Contains(rec.Body.String(), "<domain>"+d+"</domain>") {
t.Errorf("%s: %d %s", d, rec.Code, rec.Body.String())
}
}
}
// Failure is loud (issue 1631 AC2): a missing variable stops the program and its name is in the message.
func TestAMissingVariableStopsTheProgramNamingIt(t *testing.T) {
for name := range capturedEnvironment {
t.Run(name, func(t *testing.T) {
env := map[string]string{}
for k, v := range capturedEnvironment {
env[k] = v
}
delete(env, name)
if _, err := configFrom(getenvOf(env)); err == nil || !strings.Contains(err.Error(), name) {
t.Fatalf("configFrom without %s: %v", name, err)
}
var out bytes.Buffer
if code := run(getenvOf(env), &out, func(*http.Server) error { t.Fatal("served without its configuration"); return nil }); code == 0 || !strings.Contains(out.String(), name) {
t.Errorf("run without %s: exit %d, said %q", name, code, out.String())
}
})
}
}
func TestAPortThatIsNotANumberIsRefusedNamingIt(t *testing.T) {
env := map[string]string{}
for k, v := range capturedEnvironment {
env[k] = v
}
env["AMX_SMTP_PORT"] = "submission"
if _, err := configFrom(getenvOf(env)); err == nil || !strings.Contains(err.Error(), "AMX_SMTP_PORT") {
t.Fatalf("got %v", err)
}
}
+290
View File
@@ -0,0 +1,290 @@
// automx answers mail clients' autoconfiguration requests for the domains mailu serves: Thunderbird's
// /mail/config-v1.1.xml, Outlook's /autodiscover/autodiscover.xml and Apple's /mobileconfig. It replaces the
// Python automx2 image (novox/hq ADR 0480 clauses 1, 9, 10): standard library only, no database, every answer
// made from mailu's environment at the start, so a change of the domains, servers or ports reaches clients when
// the program is started again.
//
// The answers are those automx2 2021.6 gave, captured in testdata/captured (CAPTURED, issue 1630), byte for byte,
// but for the one thing testdata/captured/DIFFERENCES lists: the IMAP server is named, from AMX_IMAP_ADDRESS.
package main
import (
"encoding/xml"
"errors"
"fmt"
"html"
"io"
"net/http"
"os"
"strconv"
"strings"
"time"
)
// The environment, all required: a program that cannot say which servers clients use must not run (failure is
// loud). The names are those of mailu's files/mailu.env.
const (
envDomains = "AMX_MAIL_DOMAINS"
envImapHost = "AMX_IMAP_ADDRESS"
envImapPort = "AMX_IMAP_PORT"
envSmtpHost = "AMX_SMTP_ADDRESS"
envSmtpPort = "AMX_SMTP_PORT"
envListen = "AUTOMX_PORT"
)
type config struct {
domains []string
imapHost, smtpHost string
imapPort, smtpPort int
listen int
}
// configFrom reads the environment; it names every variable that is missing or wrong.
func configFrom(getenv func(string) string) (config, error) {
var c config
var problems []string
text := func(name string) string {
v := strings.TrimSpace(getenv(name))
if v == "" {
problems = append(problems, name+" is not set")
}
return v
}
port := func(name string) int {
v := text(name)
if v == "" {
return 0
}
n, err := strconv.Atoi(v)
if err != nil || n < 1 || n > 65535 {
problems = append(problems, fmt.Sprintf("%s is %q, not a port number", name, v))
return 0
}
return n
}
// automx2's add-domains split the list at commas and spaces.
for _, d := range strings.FieldsFunc(text(envDomains), func(r rune) bool { return r == ',' || r == ' ' }) {
c.domains = append(c.domains, strings.ToLower(d))
}
c.imapHost, c.imapPort = text(envImapHost), port(envImapPort)
c.smtpHost, c.smtpPort = text(envSmtpHost), port(envSmtpPort)
c.listen = port(envListen)
if len(problems) > 0 {
return config{}, errors.New(strings.Join(problems, "; "))
}
return c, nil
}
// What automx2 sent (Werkzeug), copied from the capture.
const (
htmlType = "text/html; charset=utf-8"
xmlType = "application/xml"
notFound = `<!doctype html>
<html lang=en>
<title>404 Not Found</title>
<h1>Not Found</h1>
<p>The requested URL was not found on the server. If you entered the URL manually please check your spelling and try again.</p>
`
notAllowed = `<!doctype html>
<html lang=en>
<title>405 Method Not Allowed</title>
<h1>Method Not Allowed</h1>
<p>The method is not allowed for the requested URL.</p>
`
internalError = `<!doctype html>
<html lang=en>
<title>500 Internal Server Error</title>
<h1>Internal Server Error</h1>
<p>The server encountered an internal error and was unable to complete your request. Either the server is overloaded or there is an error in the application.</p>
`
badRequest = `<!doctype html>
<html lang=en>
<title>400 Bad Request</title>
<h1>Bad Request</h1>
<p>The browser (or proxy) sent a request that this server could not understand.</p>
`
index = `<html><body>Show Thunderbird-style <a href="/mail/config-v1.1.xml?emailaddress=abc@example.com">XML configuration</a> for abc@example.com.</body></html>`
)
func reply(w http.ResponseWriter, status int, contentType, body string) {
w.Header().Set("Content-Type", contentType)
if status != http.StatusNoContent {
w.Header().Set("Content-Length", strconv.Itoa(len(body)))
}
w.WriteHeader(status)
if status != http.StatusNoContent {
io.WriteString(w, body)
}
}
func missing(w http.ResponseWriter, argument string) {
reply(w, http.StatusBadRequest, htmlType, fmt.Sprintf("Missing request argument %q", argument))
}
type handler struct{ config }
func newHandler(c config) http.Handler { return handler{c} }
func (h handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
var allow string
switch r.URL.Path {
case "/", "/mail/config-v1.1.xml", "/mobileconfig/":
allow = "GET, HEAD, OPTIONS"
case "/autodiscover/autodiscover.xml":
allow = "POST, OPTIONS"
case "/mobileconfig":
h.redirectToSlash(w, r)
return
default:
reply(w, http.StatusNotFound, htmlType, notFound)
return
}
if r.Method == http.MethodOptions {
w.Header().Set("Allow", allow)
reply(w, http.StatusOK, htmlType, "")
return
}
if !strings.Contains(", "+allow+",", ", "+r.Method+",") {
w.Header().Set("Allow", allow)
reply(w, http.StatusMethodNotAllowed, htmlType, notAllowed)
return
}
switch r.URL.Path {
case "/":
reply(w, http.StatusOK, htmlType, index)
case "/mail/config-v1.1.xml":
h.byAddress(w, r.URL.Query().Get("emailaddress"), r.URL.Query().Has("emailaddress"), "emailaddress", h.thunderbird)
case "/mobileconfig/":
h.byAddress(w, r.URL.Query().Get("emailaddress"), r.URL.Query().Has("emailaddress"), "emailaddress", h.apple)
case "/autodiscover/autodiscover.xml":
h.outlook(w, r)
}
}
// byAddress is the part the three protocols share: the address is required, and a domain that is not served is
// answered with no content.
func (h handler) byAddress(w http.ResponseWriter, address string, present bool, argument string, answer func(http.ResponseWriter, int, string)) {
if !present {
missing(w, argument)
return
}
domain := strings.ToLower(address[strings.LastIndex(address, "@")+1:])
for i, d := range h.domains {
if strings.Contains(address, "@") && d == domain {
answer(w, i, d)
return
}
}
reply(w, http.StatusNoContent, htmlType, "")
}
// thunderbird is Mozilla's autoconfiguration, config-v1.1. The provider id is automx2's: the provider rows were
// numbered 1, 11, 21 ... by add-domains.
func (h handler) thunderbird(w http.ResponseWriter, i int, domain string) {
d := html.EscapeString(domain)
server := func(kind, host string, port int) string {
tag := map[string]string{"imap": "incomingServer", "smtp": "outgoingServer"}[kind]
return fmt.Sprintf(`<%s type="%s"><hostname>%s</hostname><port>%d</port><socketType>STARTTLS</socketType>`+
`<username>%%EMAILADDRESS%%</username><authentication>password-cleartext</authentication></%s>`,
tag, kind, html.EscapeString(host), port, tag)
}
body := fmt.Sprintf(`<clientConfig version="1.1"><emailProvider id="automx2-%d"><identity /><domain>%s</domain>`+
`<displayName>%s</displayName><displayShortName>%s</displayShortName>%s%s</emailProvider></clientConfig>`,
1+10*i, d, d, d, server("imap", h.imapHost, h.imapPort), server("smtp", h.smtpHost, h.smtpPort))
reply(w, http.StatusOK, xmlType, body)
}
// apple and outlook answer a served domain as automx2 did: with an error (DIFFERENCES, issue 1691). They are
// kept as captured here, so the move to a Go program changes the answer only where clause 9 says.
func (h handler) apple(w http.ResponseWriter, _ int, _ string) {
reply(w, http.StatusBadRequest, htmlType, badRequest)
}
// redirectToSlash is Werkzeug's strict-slash redirect, to the name the client used.
func (h handler) redirectToSlash(w http.ResponseWriter, r *http.Request) {
scheme := "http"
if p := strings.TrimSpace(strings.Split(r.Header.Get("X-Forwarded-Proto"), ",")[0]); p != "" {
scheme = p
}
host := r.Host
if f := strings.TrimSpace(strings.Split(r.Header.Get("X-Forwarded-Host"), ",")[0]); f != "" {
host = f
}
target := scheme + "://" + host + "/mobileconfig/"
if r.URL.RawQuery != "" {
target += "?" + r.URL.RawQuery
}
w.Header().Set("Location", target)
e := html.EscapeString(target)
reply(w, http.StatusPermanentRedirect, htmlType, fmt.Sprintf(`<!doctype html>
<html lang=en>
<title>Redirecting...</title>
<h1>Redirecting...</h1>
<p>You should be redirected automatically to the target URL: <a href="%s">%s</a>. If not, click the link.
`, e, e))
}
// outlook reads the request's EMailAddress anywhere in the document, whatever the namespace. A body that is not
// one XML document is automx2's 500.
func (h handler) outlook(w http.ResponseWriter, r *http.Request) {
dec := xml.NewDecoder(io.LimitReader(r.Body, 1<<20))
var address string
var found, elements, inAddress = false, 0, false
for {
tok, err := dec.Token()
if err == io.EOF {
break
}
if err != nil {
reply(w, http.StatusInternalServerError, htmlType, internalError)
return
}
switch t := tok.(type) {
case xml.StartElement:
elements++
if t.Name.Local == "EMailAddress" && !found {
inAddress = true
}
case xml.CharData:
if inAddress {
address += string(t)
}
case xml.EndElement:
if inAddress && t.Name.Local == "EMailAddress" {
inAddress, found = false, true
}
}
}
if elements == 0 {
reply(w, http.StatusInternalServerError, htmlType, internalError)
return
}
h.byAddress(w, strings.TrimSpace(address), found, "EMailAddress", func(w http.ResponseWriter, _ int, _ string) {
reply(w, http.StatusInternalServerError, htmlType, internalError)
})
}
// run is the program: configuration from the environment, then serving. A configuration that cannot be read is
// said, with the variables' names, and the exit code is not zero.
func run(getenv func(string) string, out io.Writer, serve func(*http.Server) error) int {
c, err := configFrom(getenv)
if err != nil {
fmt.Fprintf(out, "automx: %v\n", err)
return 1
}
srv := &http.Server{
Addr: ":" + strconv.Itoa(c.listen),
Handler: newHandler(c),
ReadHeaderTimeout: 10 * time.Second,
}
if err := serve(srv); err != nil {
fmt.Fprintf(out, "automx: %v\n", err)
return 1
}
return 0
}
func main() {
os.Exit(run(os.Getenv, os.Stderr, func(s *http.Server) error { return s.ListenAndServe() }))
}
+202
View File
@@ -0,0 +1,202 @@
package main
import (
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
)
// The digest route-proxy's `trust` artifact runs: one alpine image is held for both (ADR 0480 clause 3). It was
// read from mesh-controller's modules/route-proxy/module.json and is the one hello-web runs.
const alpineDigest = "alpine@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b"
type manifest struct {
Tools []string `json:"tools"`
Resources []map[string]any `json:"resources"`
Listens []struct {
Name string `json:"name"`
Port int `json:"port"`
} `json:"listens"`
Data struct {
Own []struct {
ID string `json:"id"`
Path string `json:"path"`
} `json:"own"`
} `json:"data"`
Build map[string]any `json:"build"`
}
func readManifest(t *testing.T) manifest {
t.Helper()
var m manifest
if err := json.Unmarshal([]byte(readFile(t, "../../module.json")), &m); err != nil {
t.Fatal(err)
}
return m
}
func (m manifest) resource(id string) map[string]any {
for _, r := range m.Resources {
if r["id"] == id {
return r
}
}
return nil
}
func (m manifest) artifact(name string) map[string]any {
arts, _ := m.Build["artifacts"].([]any)
for _, a := range arts {
if a, _ := a.(map[string]any); a["name"] == name {
return a
}
}
return nil
}
func strs(v any) []string {
var out []string
l, _ := v.([]any)
for _, e := range l {
out = append(out, e.(string))
}
return out
}
// Clause 2: a Go bundle from a package of the module's own go.mod, stamped for alpine; an archive resource unpacks
// it; the container mounts that read-only at /opt/mesh, calls the program there and names the archive in restart-on.
func TestAutomxIsAGoBundleForAlpineMountedAtOptMeshAndRestartedWithIt(t *testing.T) {
m := readManifest(t)
b := m.artifact("automx")
if b == nil || b["kind"] != "bundle" || b["language"] != "go" || b["system"] != "alpine" || b["from"] != "cmd/automx" || b["binary"] != "automx" {
t.Fatalf("artifact automx is %v, want a Go bundle, system alpine, from cmd/automx, binary automx", b)
}
if _, loads := b["loads"]; loads {
t.Errorf("a program run in a container is not loaded by the tool runner: %v", b["loads"])
}
goMod := readFile(t, "../../go.mod")
if !strings.HasPrefix(goMod, "module mailu\n") {
t.Errorf("the module's go.mod is %q; the bundle is a package of it", goMod)
}
if _, err := os.Stat(filepath.Join("..", "..", filepath.FromSlash(b["from"].(string)), "main.go")); err != nil {
t.Errorf("the bundle's package is not in the module's directory: %v", err)
}
unpack := m.resource("program-automx")
if unpack == nil || unpack["type"] != "archive" || unpack["artifact"] != "automx" || unpack["unknown"] != "program" {
t.Fatalf("resource program-automx is %v, want an archive of the bundle", unpack)
}
where, _ := unpack["path"].(string)
c := m.resource("automx")
if c == nil || c["type"] != "container" {
t.Fatalf("no container automx: %v", c)
}
if got := strs(c["volumes"]); len(got) != 1 || got[0] != where+":/opt/mesh:ro" {
t.Errorf("volumes %v, want the archive's place %q mounted read-only at /opt/mesh", got, where)
}
if got := strs(c["args"]); len(got) != 1 || got[0] != "/opt/mesh/automx" {
t.Errorf("args %v, want the program at /opt/mesh/automx", got)
}
if !contains(strs(c["restart-on"]), "program-automx") {
t.Errorf("restart-on %v does not name the archive: a new program would leave the old one running", c["restart-on"])
}
if !contains(strs(c["restart-on"]), "config-env") {
t.Errorf("restart-on %v does not name the environment file the program reads at its start", c["restart-on"])
}
}
// Clause 3: an upstream artifact pinned by digest, the one route-proxy's trust artifact runs; never an image
// built from a Dockerfile (clause 11), and no Dockerfile or base to build one on.
func TestAutomxRunsTheMirroredAlpineImageAndBuildsNoImage(t *testing.T) {
m := readManifest(t)
c := m.resource("automx")
base := m.artifact(c["artifact"].(string))
if base == nil || base["kind"] != "upstream" || base["from"] != alpineDigest {
t.Errorf("the container runs artifact %v, want an upstream artifact %s", base, alpineDigest)
}
if _, has := c["image"]; has {
t.Errorf("the container names an image %v, fetched from a registry on the machine", c["image"])
}
for _, a := range m.Build["artifacts"].([]any) {
if a.(map[string]any)["kind"] == "image" {
t.Errorf("artifact %v is an image built from a Dockerfile (ADR 0452)", a)
}
}
if _, has := m.Build["on"]; has {
t.Errorf("build.on %v: nothing is built on a base image any more", m.Build["on"])
}
for _, gone := range []string{"../../automx/Dockerfile", "../../automx"} {
if _, err := os.Stat(gone); err == nil {
t.Errorf("%s is still there", gone)
}
}
}
// Clause 1: the container stays as it was for what watches it.
func TestAutomxKeepsItsContainerNameNetworkPortAndEnvironment(t *testing.T) {
m := readManifest(t)
c := m.resource("automx")
if c["name"] != "mailu-automx" || c["network"] != "mailu" {
t.Errorf("container %v on network %v, want mailu-automx on mailu", c["name"], c["network"])
}
if got := strs(c["ports"]); len(got) != 1 || got[0] != "4243" {
t.Errorf("ports %v, want 4243", got)
}
if got := strs(c["env-file"]); len(got) != 1 || got[0] != "${dir:state}/mailu.env" {
t.Errorf("env-file %v, want mailu's environment only", got)
}
if !strings.Contains("\n"+readFile(t, "../../files/mailu.env"), "\nAUTOMX_PORT=4243\n") {
t.Errorf("mailu.env does not say AUTOMX_PORT=4243")
}
for _, l := range m.Listens {
if l.Name == "autoconfig" && l.Port == 4243 {
return
}
}
t.Errorf("the autoconfig endpoint is not on 4243: %v", m.Listens)
}
// Clause 10: a database nobody reads is gone from the declaration (the file stays on disk), and the container says
// whether it answers with one captured request and its captured answer.
func TestAutomxKeepsNoDatabaseAndSaysWhetherItAnswers(t *testing.T) {
m := readManifest(t)
if m.resource("data-automx") != nil {
t.Errorf("the data-automx directory is still declared")
}
for _, it := range m.Data.Own {
if strings.Contains(it.ID, "automx") || strings.Contains(it.Path, "data-automx") {
t.Errorf("data item %v is still declared", it)
}
}
for _, tool := range m.Tools {
if tool == "automx_rows" {
t.Errorf("the tool automx_rows reads a database that no longer exists (issue 1700)")
}
}
h, _ := m.resource("automx")["health"].(map[string]any)
if h["kind"] != "http" || h["endpoint"] != "autoconfig" {
t.Fatalf("health %v, want an http check of the autoconfig endpoint", h)
}
// The request is captured fixture 06 and the answer its status and body.
rq := readRequest(t, filepath.Join(captured, "06-autoconfig-root"))
if u := strings.SplitN(rq.url, "/", 4); "/"+u[len(u)-1] != h["path"] || rq.method != "GET" {
t.Errorf("health path %v is not captured request %s %s", h["path"], rq.method, rq.url)
}
if st := readFile(t, filepath.Join(captured, "06-autoconfig-root.status")); !strings.HasPrefix(st, "200 ") || h["status"] != float64(200) {
t.Errorf("health status %v, captured %q", h["status"], st)
}
if body, _ := h["body"].(string); body == "" || !strings.Contains(readFile(t, filepath.Join(captured, "06-autoconfig-root.body")), body) {
t.Errorf("health body %q is not in the captured answer", h["body"])
}
}
func contains(l []string, s string) bool {
for _, e := range l {
if e == s {
return true
}
}
return false
}
@@ -0,0 +1 @@
<clientConfig version="1.1"><emailProvider id="automx2-1"><identity /><domain>novox.be</domain><displayName>novox.be</displayName><displayShortName>novox.be</displayShortName><incomingServer type="imap"><hostname>mail.novox.be</hostname><port>143</port><socketType>STARTTLS</socketType><username>%EMAILADDRESS%</username><authentication>password-cleartext</authentication></incomingServer><outgoingServer type="smtp"><hostname>mail.novox.be</hostname><port>587</port><socketType>STARTTLS</socketType><username>%EMAILADDRESS%</username><authentication>password-cleartext</authentication></outgoingServer></emailProvider></clientConfig>
@@ -0,0 +1 @@
<clientConfig version="1.1"><emailProvider id="automx2-1"><identity /><domain>novox.be</domain><displayName>novox.be</displayName><displayShortName>novox.be</displayShortName><incomingServer type="imap"><hostname>mail.novox.be</hostname><port>143</port><socketType>STARTTLS</socketType><username>%EMAILADDRESS%</username><authentication>password-cleartext</authentication></incomingServer><outgoingServer type="smtp"><hostname>mail.novox.be</hostname><port>587</port><socketType>STARTTLS</socketType><username>%EMAILADDRESS%</username><authentication>password-cleartext</authentication></outgoingServer></emailProvider></clientConfig>
@@ -0,0 +1 @@
<clientConfig version="1.1"><emailProvider id="automx2-1"><identity /><domain>novox.be</domain><displayName>novox.be</displayName><displayShortName>novox.be</displayShortName><incomingServer type="imap"><hostname>mail.novox.be</hostname><port>143</port><socketType>STARTTLS</socketType><username>%EMAILADDRESS%</username><authentication>password-cleartext</authentication></incomingServer><outgoingServer type="smtp"><hostname>mail.novox.be</hostname><port>587</port><socketType>STARTTLS</socketType><username>%EMAILADDRESS%</username><authentication>password-cleartext</authentication></outgoingServer></emailProvider></clientConfig>
+12 -1
View File
@@ -14,6 +14,13 @@ Each capture is four files with the same stem:
The files were written from the tool's answers by a script (not by retyping each by hand); the tool returns JSON, so
a later capture that wants exact bytes needs a tool that writes the answer to a file.
A fixture is not the bytes on the wire in four ways (issue 1701):
- Date is left out of .headers (it changes on every request).
- .headers are sorted by name.
- .status holds the tool's status line without the protocol (the tool reported "HTTP/1.1" apart).
- Header names are as the tool's Go HTTP client gives them (canonical case, after the mesh's route), not as
automx wrote them.
What the answers show (read, not guessed):
- GET /mail/config-v1.1.xml?emailaddress=... answers 200 application/xml on all three names; same body on each.
- The incoming (IMAP) server has an EMPTY hostname (<hostname />), port 143, STARTTLS. The outgoing server is
@@ -27,4 +34,8 @@ What the answers show (read, not guessed):
- GET /mobileconfig?emailaddress=... redirects (308) to /mobileconfig/?emailaddress=...; for the served domain that
answers 400 with no profile; unknown domain 204; no emailaddress 400.
- GET / on autoconfig answers 200 with a small HTML page linking to an example address.
- Not captured: the database rows (no mesh tool reads /data/db.sqlite; issue 1630 AC4).
- The database rows behind the answers were read once with the tool automx_rows (mailu, issue 1630 AC4), which
ran sqlite3 read-only inside mailu-automx. The tool is retired with the database (ADR 0480 clause 10, issue
1700), so nothing reads the rows any more; the answers above are what they gave.
- The Go program that replaced automx2 answers these requests as they are captured here, but for what
DIFFERENCES lists.
+25
View File
@@ -0,0 +1,25 @@
How the Go program (modules/mailu/cmd/automx) answers differently from the automx2 2021.6 answers captured here
(issue 1631, ADR 0480 clause 9). automx_test.go answers every .request and compares with the capture; this is the
whole list of where the two differ.
1. The IMAP server's name (the one correction clause 9 allows). automx2 answered <hostname /> for the incoming
server because its setup script read AMX_IMAP_SERVER, which mailu's environment does not set (it sets
AMX_IMAP_ADDRESS). The program names it from AMX_IMAP_ADDRESS (and the port from AMX_IMAP_PORT).
Fixtures 01, 02 and 03 each have a <stem>.corrected.body: the capture with <hostname /> replaced by
<hostname>mail.novox.be</hostname> and nothing else (a test checks that). Their Content-Length is that file's
size, 634, not the captured 612.
2. The Server header. The capture has "Server: Werkzeug/3.1.9 Python/3.9.5". The program sends no Server header, and
Date, which the captured .headers leave out, is sent as net/http sends it.
Not changed, though issue 1691 calls them faults; they are kept as captured because clause 9 allows only the first
correction, and their cause is not known from the capture:
- Outlook's POST for a served domain (fixture 08) answers 500.
- Apple's /mobileconfig/ for a served domain (fixture 15) answers 400 with no profile.
Whether automx2 failed there because of the empty IMAP name is a guess; it can be tested only against the
running program after the delivery. Issue 1691 stays open for them.
Assumptions the fixtures do not settle (answers to requests nobody captured): a domain is compared in lower case;
an address without "@" is an unknown domain (204); HEAD is answered as GET and OPTIONS with the Allow header and
no body; the redirect of /mobileconfig takes its scheme and host from X-Forwarded-Proto and X-Forwarded-Host when
present (the route sends the client's, as fixture 14's Location shows) and from the request otherwise.
+3
View File
@@ -0,0 +1,3 @@
module mailu
go 1.22
+34 -24
View File
@@ -71,8 +71,7 @@
"mailu_list_domains",
"mailu_read_mail",
"mailu_search_mail",
"mailu_health",
"automx_rows"
"mailu_health"
],
"emits": [
"user.created",
@@ -216,12 +215,6 @@
"class": "rebuildable",
"why": "certificates, issued again"
},
{
"id": "automx",
"path": "${dir:data-automx}",
"class": "rebuildable",
"why": "client autoconfiguration, written again"
},
{
"id": "fetchmail",
"path": "${dir:data-fetchmail}",
@@ -308,12 +301,6 @@
"mode": "0700",
"unknown": "program"
},
{
"id": "data-automx",
"type": "directory",
"mode": "0700",
"unknown": "program"
},
{
"id": "config-env",
"type": "file",
@@ -681,11 +668,25 @@
"takes-config": ["domain", "proxy-address", "sitename", "website", "url", "container"],
"merge": "json"
},
{
"id": "program-automx",
"type": "archive",
"path": "${dir:state}/automx",
"artifact": "automx",
"unknown": "program"
},
{
"id": "automx",
"type": "container",
"name": "mailu-automx",
"artifact": "automx",
"artifact": "alpine",
"health": {
"kind": "http",
"endpoint": "autoconfig",
"path": "/",
"status": 200,
"body": "Show Thunderbird-style"
},
"network": "mailu",
"env-file": [
"${dir:state}/mailu.env"
@@ -694,17 +695,18 @@
"4243"
],
"volumes": [
"${dir:data-automx}:/data"
"${dir:state}/automx:/opt/mesh:ro"
],
"args": [
"/opt/mesh/automx"
],
"restart-on": [
"program-automx",
"config-env"
]
}
],
"build": {
"on": [
{
"arg": "PYTHON_BASE",
"image": "python@sha256:25f3cfeaceca14921366af4d1240b56457ef46273bdb508c7b0e8f469f6fd228"
}
],
"artifacts": [
{
"name": "code",
@@ -731,8 +733,16 @@
},
{
"name": "automx",
"kind": "image",
"from": "automx/Dockerfile"
"kind": "bundle",
"language": "go",
"system": "alpine",
"from": "cmd/automx",
"binary": "automx"
},
{
"name": "alpine",
"kind": "upstream",
"from": "alpine@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b"
},
{
"name": "files",
-103
View File
@@ -1,103 +0,0 @@
// What automx_rows reads (novox/hq issue 1630): the rows of automx2's database as stored, through a read-only
// sqlite3 inside its container. The database here is a real SQLite file made by the test with the tables
// automx/files/setup-db creates and the rows add-domains writes — with the IMAP server's name empty, the case
// the live instance is in. The fake container runs each command's statement against that database and prints
// it the way the image's sqlite3 does in list mode (one line per row, cells apart by "|", nothing for no rows); that sqlite3
// refuses `-json` ("sqlite3: Error: unknown option: -json", captured from novox/mailu.automx_rows, issue 1630). Run against the compiled module.
import { test } from "node:test";
import assert from "node:assert/strict";
import { DatabaseSync } from "node:sqlite";
import { readAutomx, readCommand, selectOf, parseRows, AUTOMX_DATABASE } from "../dist/automx.js";
function database(imapName: string): DatabaseSync {
const db = new DatabaseSync(":memory:");
db.exec(`
CREATE TABLE provider(id INT PRIMARY KEY NOT NULL, name TEXT NOT NULL, short_name TEXT NOT NULL);
CREATE TABLE server(id INT PRIMARY KEY NOT NULL, prio INT NOT NULL DEFAULT 10, name TEXT NOT NULL, port INT NOT NULL,
type TEXT NOT NULL, socket_type TEXT NOT NULL, user_name TEXT NOT NULL, authentication TEXT NOT NULL);
CREATE TABLE domain(id INT PRIMARY KEY NOT NULL, name TEXT NOT NULL, provider_id INT NOT NULL, ldapserver_id INT NULL);
CREATE TABLE server_domain(server_id INT NOT NULL, domain_id INT NOT NULL);
INSERT INTO provider VALUES (1, 'example.org', 'example.org');
INSERT INTO server(id, port, type, name, socket_type, user_name, authentication)
VALUES (2, 143, 'imap', '${imapName}', 'STARTTLS', '%EMAILADDRESS%', 'password-cleartext');
INSERT INTO server(id, port, type, name, socket_type, user_name, authentication)
VALUES (3, 587, 'smtp', 'mail.example.org', 'STARTTLS', '%EMAILADDRESS%', 'password-cleartext');
INSERT INTO domain VALUES (5, 'example.org', 1, NULL);
INSERT INTO server_domain VALUES (2, 5), (3, 5);
`);
return db;
}
function container(db: DatabaseSync, asked: string[][] = []) {
return async (_name: string, command: string[]) => {
asked.push(command);
if (command.includes("-json")) throw Object.assign(new Error("Command failed: docker exec"), { stderr: "sqlite3: Error: unknown option: -json\n" });
const rows = db.prepare(command[command.length - 1]).all() as unknown as Record<string, string>[];
return rows.length === 0 ? "" : rows.map((r) => Object.values(r).join("|")).join("\n") + "\n";
};
}
test("the rows come back as stored, and an empty server name is said", async () => {
const answer = await readAutomx(container(database("")), "mailu-automx");
assert.equal(answer.database, AUTOMX_DATABASE);
assert.deepEqual(answer.unreadable, {});
assert.deepEqual(answer.tables.provider, [{ id: 1, name: "example.org", short_name: "example.org" }]);
assert.deepEqual(answer.tables.domain, [{ id: 5, name: "example.org", provider_id: 1, ldapserver_id: null }]);
assert.deepEqual(answer.tables.server_domain, [{ server_id: 2, domain_id: 5 }, { server_id: 3, domain_id: 5 }]);
assert.deepEqual(
answer.tables.server.map((s) => [s.id, s.type, s.name, s.port]),
[[2, "imap", "", 143], [3, "smtp", "mail.example.org", 587]],
);
assert.deepEqual(answer.empty_names, [{ id: 2, type: "imap" }]);
});
test("a named IMAP server is not reported empty", async () => {
const answer = await readAutomx(container(database("mail.example.org")), "mailu-automx");
assert.deepEqual(answer.empty_names, []);
});
test("every command opens the file read-only and runs one fixed SELECT", async () => {
const asked: string[][] = [];
await readAutomx(container(database("x"), asked), "mailu-automx");
assert.equal(asked.length, 4);
for (const c of asked) {
assert.deepEqual(c.slice(0, 3), ["sqlite3", "-readonly", AUTOMX_DATABASE]);
assert.match(c[3], /^SELECT /);
assert.equal(c.length, 4);
assert.ok(!c.includes("-json"));
}
assert.deepEqual(readCommand("provider"), ["sqlite3", "-readonly", AUTOMX_DATABASE, selectOf("provider")]);
});
test("a table that cannot be read is said and does not hide the others", async () => {
const db = database("x");
const exec = async (name: string, command: string[]) => {
if (command[3].includes("FROM domain ")) throw Object.assign(new Error("Command failed: docker exec"), { stderr: "Error: unable to open database file\nmore" });
return container(db)(name, command);
};
const answer = await readAutomx(exec, "mailu-automx");
assert.deepEqual(answer.unreadable, { domain: "Error: unable to open database file" });
assert.equal(answer.tables.provider.length, 1);
assert.equal(answer.tables.domain, undefined);
});
test("a NULL, an empty text, a number and a text holding a bar or a newline are read apart", () => {
const hex = (t: string) => Buffer.from(t).toString("hex").toUpperCase();
const line = [`integer:${hex("5")}`, `text:${hex("a|b\nc")}`, "text:", "null:"].join("|");
const rows = parseRows("domain", line + "\n");
assert.deepEqual(rows, [{ id: 5, name: "a|b\nc", provider_id: "", ldapserver_id: null }]);
});
test("output that is not cells is an unreadable table, not a guess", async () => {
const exec = async () => "Error: something else\n";
const answer = await readAutomx(exec, "mailu-automx");
assert.equal(Object.keys(answer.unreadable).length, 4);
});
test("the image's refusal of -json is said as the table's reason", async () => {
const exec = async (_n: string, command: string[]) => container(database("x"))("", ["sqlite3", "-json", ...command.slice(1)]);
const answer = await readAutomx(exec, "mailu-automx");
assert.deepEqual(answer.unreadable.domain, "sqlite3: Error: unknown option: -json");
});
+1 -2
View File
@@ -135,7 +135,6 @@ test("the health tool is served without the admin API, on the smtp container it
await import("../dist/tools/index.js");
const mailu = collectTools({ MESH_MAILU_SMTP_CONTAINER: "mailu-smtp" }).find((m) => m.module === "mailu");
const names = (mailu?.tools ?? []).map((t) => t.name);
assert.deepEqual(names, ["mailu_health", "automx_rows"]);
assert.deepEqual(names, ["mailu_health"]);
assert.match(mailu!.tools[0].description, /inside mailu-smtp/);
assert.match(mailu!.tools[1].description, /read-only, inside mailu-automx/);
});
+1 -19
View File
@@ -4,7 +4,6 @@
import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools";
import { MailuClient } from "../client.js";
import { readAutomx } from "../automx.js";
import { dockerExec, mailFlows, DEFERRED_BOUND_SECONDS, PUBLIC_NAME, SPREAD_DOMAINS } from "../health.js";
/**
@@ -27,23 +26,6 @@ export function getHealthTool(env: NodeJS.ProcessEnv): ToolDefinition {
};
}
/**
* What automx holds, as stored (novox/hq issue 1630): the rows behind the answers it gives mail clients. It needs
* no admin API, only the automx container, so like the health tool it is served without the API.
*/
export function getAutomxTool(env: NodeJS.ProcessEnv): ToolDefinition {
const container = env.MESH_MAILU_AUTOMX_CONTAINER || "mailu-automx";
return {
name: "automx_rows",
description:
`Read the autoconfiguration module's database, as stored, read-only, inside ${container}: its provider, ` +
"server, domain and server_domain rows, and the servers whose name is empty (a mail client is then told " +
"an empty host name). Writes nothing.",
input: {},
run: async () => readAutomx(dockerExec(), container),
};
}
export function getMailuTools(mailu: MailuClient): ToolDefinition[] {
return [
{
@@ -191,5 +173,5 @@ registerModuleTools("mailu", (env) => {
} catch {
tools = [];
}
return [...tools, getHealthTool(env), getAutomxTool(env)];
return [...tools, getHealthTool(env)];
});
+1 -1
View File
@@ -8,5 +8,5 @@
"skipLibCheck": true,
"noEmit": true
},
"include": ["client.ts", "health.ts", "automx.ts", "index.ts", "tools/index.ts"]
"include": ["client.ts", "health.ts", "index.ts", "tools/index.ts"]
}