From 1f7dd702d283e647d4189737601d80ebb347ae5e Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 21 Oct 2026 16:24:58 +0200 Subject: [PATCH] Read automx's rows with a mode its image's sqlite3 knows, since it refuses -json (issue 1630) --- modules/mailu/automx.ts | 64 ++++++++++++++++++-------- modules/mailu/test/automx-rows.test.ts | 40 ++++++++++++---- 2 files changed, 77 insertions(+), 27 deletions(-) diff --git a/modules/mailu/automx.ts b/modules/mailu/automx.ts index 9af3b039..1512f0b9 100644 --- a/modules/mailu/automx.ts +++ b/modules/mailu/automx.ts @@ -7,14 +7,25 @@ import type { Exec } from "./health.js"; export const AUTOMX_DATABASE = "/data/db.sqlite"; -/** The tables read, each with the one query that reads it. Fixed text: the tool takes no query from its caller. */ -export const AUTOMX_TABLES: Record = { - provider: "SELECT id, name, short_name FROM provider ORDER BY id", - server: "SELECT id, prio, name, port, type, socket_type, user_name, authentication FROM server ORDER BY id", - domain: "SELECT id, name, provider_id, ldapserver_id FROM domain ORDER BY id", - server_domain: "SELECT server_id, domain_id FROM server_domain ORDER BY domain_id, server_id", +/** The tables read, each with its columns and its ordering. Fixed text: the tool takes no query from its caller. */ +export const AUTOMX_TABLES: Record = { + provider: { columns: ["id", "name", "short_name"], order: "id" }, + server: { columns: ["id", "prio", "name", "port", "type", "socket_type", "user_name", "authentication"], order: "id" }, + domain: { columns: ["id", "name", "provider_id", "ldapserver_id"], order: "id" }, + server_domain: { columns: ["server_id", "domain_id"], order: "domain_id, server_id" }, }; +/** + * The one statement that reads a table. The sqlite3 in the mailu-automx image does not know `-json` (it answers + * "sqlite3: Error: unknown option: -json", issue 1630), so each column is printed as `:`, which + * the oldest sqlite3 prints in its default list mode: no quoting, no separator or newline inside a cell, and a NULL + * (`null:`) apart from an empty text (`text:`). + */ +export function selectOf(table: string): string { + const t = AUTOMX_TABLES[table]; + return `SELECT ${t.columns.map((c) => `typeof(${c}) || ':' || hex(${c})`).join(", ")} FROM ${table} ORDER BY ${t.order}`; +} + export type Row = Record; export type AutomxRows = { @@ -26,26 +37,43 @@ export type AutomxRows = { unreadable: Record; }; -/** sqlite3 -json prints one JSON array, or nothing at all for a table without rows. */ -export function parseRows(output: string): Row[] { - const text = output.trim(); - if (text === "") return []; - const rows = JSON.parse(text) as unknown; - if (!Array.isArray(rows)) throw new Error("sqlite3 did not print a list of rows"); - return rows as Row[]; +/** One cell as selectOf prints it: `:`; text is the bytes read as UTF-8, a number is read from its text. */ +function cellOf(cell: string): string | number | null { + const at = cell.indexOf(":"); + if (at < 0 || cell.length - at - 1 > 0 && !/^([0-9A-Fa-f]{2})+$/.test(cell.slice(at + 1))) throw new Error(`sqlite3 printed ${JSON.stringify(cell.slice(0, 40))}, not a cell`); + const type = cell.slice(0, at); + const bytes = Buffer.from(cell.slice(at + 1), "hex").toString("utf8"); + switch (type) { + case "null": return null; + case "integer": case "real": return Number(bytes); + default: return bytes; + } } -/** The command that reads one table: read-only, JSON, one fixed statement. */ -export function readCommand(sql: string): string[] { - return ["sqlite3", "-readonly", "-json", AUTOMX_DATABASE, sql]; +/** sqlite3 prints one line per row, cells apart by `|`, and nothing at all for a table without rows. */ +export function parseRows(table: string, output: string): Row[] { + const columns = AUTOMX_TABLES[table].columns; + const rows: Row[] = []; + for (const line of output.split("\n")) { + if (line.trim() === "") continue; + const cells = line.replace(/\r$/, "").split("|"); + if (cells.length !== columns.length) throw new Error(`sqlite3 printed ${cells.length} cells for ${columns.length} columns`); + rows.push(Object.fromEntries(columns.map((c, i) => [c, cellOf(cells[i])]))); + } + return rows; +} + +/** The command that reads one table: read-only, one fixed statement. */ +export function readCommand(table: string): string[] { + return ["sqlite3", "-readonly", AUTOMX_DATABASE, selectOf(table)]; } /** Read every table; one that fails is said in `unreadable` and does not hide the others. */ export async function readAutomx(exec: Exec, container: string): Promise { const answer: AutomxRows = { database: AUTOMX_DATABASE, tables: {}, empty_names: [], unreadable: {} }; - for (const [table, sql] of Object.entries(AUTOMX_TABLES)) { + for (const table of Object.keys(AUTOMX_TABLES)) { try { - answer.tables[table] = parseRows(await exec(container, readCommand(sql))); + answer.tables[table] = parseRows(table, await exec(container, readCommand(table))); } catch (e) { const err = e as { stderr?: string; message?: string }; const text = (err?.stderr ?? "").trim() || (err?.message ?? String(e)).trim(); diff --git a/modules/mailu/test/automx-rows.test.ts b/modules/mailu/test/automx-rows.test.ts index b0d048f5..8ef35222 100644 --- a/modules/mailu/test/automx-rows.test.ts +++ b/modules/mailu/test/automx-rows.test.ts @@ -2,13 +2,14 @@ // sqlite3 inside its container. The database here is a real SQLite file made by the test with the tables // automx/files/setup-db creates and the rows add-domains writes — with the IMAP server's name empty, the case // the live instance is in. The fake container runs each command's statement against that database and prints -// it the way `sqlite3 -json` does (one JSON array, nothing for no rows). Run against the compiled module. +// it the way the image's sqlite3 does in list mode (one line per row, cells apart by "|", nothing for no rows); that sqlite3 +// refuses `-json` ("sqlite3: Error: unknown option: -json", captured from novox/mailu.automx_rows, issue 1630). Run against the compiled module. import { test } from "node:test"; import assert from "node:assert/strict"; import { DatabaseSync } from "node:sqlite"; -import { readAutomx, readCommand, AUTOMX_DATABASE } from "../dist/automx.js"; +import { readAutomx, readCommand, selectOf, parseRows, AUTOMX_DATABASE } from "../dist/automx.js"; function database(imapName: string): DatabaseSync { const db = new DatabaseSync(":memory:"); @@ -32,8 +33,9 @@ function database(imapName: string): DatabaseSync { function container(db: DatabaseSync, asked: string[][] = []) { return async (_name: string, command: string[]) => { asked.push(command); - const rows = db.prepare(command[command.length - 1]).all(); - return rows.length === 0 ? "" : JSON.stringify(rows) + "\n"; + if (command.includes("-json")) throw Object.assign(new Error("Command failed: docker exec"), { stderr: "sqlite3: Error: unknown option: -json\n" }); + const rows = db.prepare(command[command.length - 1]).all() as unknown as Record[]; + return rows.length === 0 ? "" : rows.map((r) => Object.values(r).join("|")).join("\n") + "\n"; }; } @@ -61,17 +63,18 @@ test("every command opens the file read-only and runs one fixed SELECT", async ( await readAutomx(container(database("x"), asked), "mailu-automx"); assert.equal(asked.length, 4); for (const c of asked) { - assert.deepEqual(c.slice(0, 4), ["sqlite3", "-readonly", "-json", AUTOMX_DATABASE]); - assert.match(c[4], /^SELECT /); - assert.equal(c.length, 5); + assert.deepEqual(c.slice(0, 3), ["sqlite3", "-readonly", AUTOMX_DATABASE]); + assert.match(c[3], /^SELECT /); + assert.equal(c.length, 4); + assert.ok(!c.includes("-json")); } - assert.deepEqual(readCommand("SELECT 1").length, 5); + assert.deepEqual(readCommand("provider"), ["sqlite3", "-readonly", AUTOMX_DATABASE, selectOf("provider")]); }); test("a table that cannot be read is said and does not hide the others", async () => { const db = database("x"); const exec = async (name: string, command: string[]) => { - if (command[4].includes("FROM domain ")) throw Object.assign(new Error("Command failed: docker exec"), { stderr: "Error: unable to open database file\nmore" }); + if (command[3].includes("FROM domain ")) throw Object.assign(new Error("Command failed: docker exec"), { stderr: "Error: unable to open database file\nmore" }); return container(db)(name, command); }; const answer = await readAutomx(exec, "mailu-automx"); @@ -79,3 +82,22 @@ test("a table that cannot be read is said and does not hide the others", async ( assert.equal(answer.tables.provider.length, 1); assert.equal(answer.tables.domain, undefined); }); + +test("a NULL, an empty text, a number and a text holding a bar or a newline are read apart", () => { + const hex = (t: string) => Buffer.from(t).toString("hex").toUpperCase(); + const line = [`integer:${hex("5")}`, `text:${hex("a|b\nc")}`, "text:", "null:"].join("|"); + const rows = parseRows("domain", line + "\n"); + assert.deepEqual(rows, [{ id: 5, name: "a|b\nc", provider_id: "", ldapserver_id: null }]); +}); + +test("output that is not cells is an unreadable table, not a guess", async () => { + const exec = async () => "Error: something else\n"; + const answer = await readAutomx(exec, "mailu-automx"); + assert.equal(Object.keys(answer.unreadable).length, 4); +}); + +test("the image's refusal of -json is said as the table's reason", async () => { + const exec = async (_n: string, command: string[]) => container(database("x"))("", ["sqlite3", "-json", ...command.slice(1)]); + const answer = await readAutomx(exec, "mailu-automx"); + assert.deepEqual(answer.unreadable.domain, "sqlite3: Error: unknown option: -json"); +});