Merge pull request 'Read automx's rows in a mode its image's sqlite3 knows (issue 1630)' (#418) from fix/1630-automx-rows-sqlite into main

modules/mailu/automx.ts

@@ -7,14 +7,25 @@ import type { Exec } from "./health.js";
 
 export const AUTOMX_DATABASE = "/data/db.sqlite";
 
-/** The tables read, each with the one query that reads it. Fixed text: the tool takes no query from its caller. */
+/** The tables read, each with its columns and its ordering. Fixed text: the tool takes no query from its caller. */
-export const AUTOMX_TABLES: Record<string, string> = {
+export const AUTOMX_TABLES: Record<string, { columns: string[]; order: string }> = {
-  provider: "SELECT id, name, short_name FROM provider ORDER BY id",
+  provider: { columns: ["id", "name", "short_name"], order: "id" },
-  server: "SELECT id, prio, name, port, type, socket_type, user_name, authentication FROM server ORDER BY id",
+  server: { columns: ["id", "prio", "name", "port", "type", "socket_type", "user_name", "authentication"], order: "id" },
-  domain: "SELECT id, name, provider_id, ldapserver_id FROM domain ORDER BY id",
+  domain: { columns: ["id", "name", "provider_id", "ldapserver_id"], order: "id" },
-  server_domain: "SELECT server_id, domain_id FROM server_domain ORDER BY domain_id, server_id",
+  server_domain: { columns: ["server_id", "domain_id"], order: "domain_id, server_id" },
 };
 
+/**
+ * The one statement that reads a table. The sqlite3 in the mailu-automx image does not know `-json` (it answers
+ * "sqlite3: Error: unknown option: -json", issue 1630), so each column is printed as `<type>:<hex of its bytes>`, which
+ * the oldest sqlite3 prints in its default list mode: no quoting, no separator or newline inside a cell, and a NULL
+ * (`null:`) apart from an empty text (`text:`).
+ */
+export function selectOf(table: string): string {
+  const t = AUTOMX_TABLES[table];
+  return `SELECT ${t.columns.map((c) => `typeof(${c}) || ':' || hex(${c})`).join(", ")} FROM ${table} ORDER BY ${t.order}`;
+}
+
 export type Row = Record<string, string | number | null>;
 
 export type AutomxRows = {
@@ -26,26 +37,43 @@ export type AutomxRows = {
   unreadable: Record<string, string>;
 };
 
-/** sqlite3 -json prints one JSON array, or nothing at all for a table without rows. */
+/** One cell as selectOf prints it: `<type>:<hex>`; text is the bytes read as UTF-8, a number is read from its text. */
-export function parseRows(output: string): Row[] {
+function cellOf(cell: string): string | number | null {
-  const text = output.trim();
+  const at = cell.indexOf(":");
-  if (text === "") return [];
+  if (at < 0 || cell.length - at - 1 > 0 && !/^([0-9A-Fa-f]{2})+$/.test(cell.slice(at + 1))) throw new Error(`sqlite3 printed ${JSON.stringify(cell.slice(0, 40))}, not a cell`);
-  const rows = JSON.parse(text) as unknown;
+  const type = cell.slice(0, at);
-  if (!Array.isArray(rows)) throw new Error("sqlite3 did not print a list of rows");
+  const bytes = Buffer.from(cell.slice(at + 1), "hex").toString("utf8");
-  return rows as Row[];
+  switch (type) {
+    case "null": return null;
+    case "integer": case "real": return Number(bytes);
+    default: return bytes;
+  }
 }
 
-/** The command that reads one table: read-only, JSON, one fixed statement. */
+/** sqlite3 prints one line per row, cells apart by `|`, and nothing at all for a table without rows. */
-export function readCommand(sql: string): string[] {
+export function parseRows(table: string, output: string): Row[] {
-  return ["sqlite3", "-readonly", "-json", AUTOMX_DATABASE, sql];
+  const columns = AUTOMX_TABLES[table].columns;
+  const rows: Row[] = [];
+  for (const line of output.split("\n")) {
+    if (line.trim() === "") continue;
+    const cells = line.replace(/\r$/, "").split("|");
+    if (cells.length !== columns.length) throw new Error(`sqlite3 printed ${cells.length} cells for ${columns.length} columns`);
+    rows.push(Object.fromEntries(columns.map((c, i) => [c, cellOf(cells[i])])));
+  }
+  return rows;
+}
+
+/** The command that reads one table: read-only, one fixed statement. */
+export function readCommand(table: string): string[] {
+  return ["sqlite3", "-readonly", AUTOMX_DATABASE, selectOf(table)];
 }
 
 /** Read every table; one that fails is said in `unreadable` and does not hide the others. */
 export async function readAutomx(exec: Exec, container: string): Promise<AutomxRows> {
   const answer: AutomxRows = { database: AUTOMX_DATABASE, tables: {}, empty_names: [], unreadable: {} };
-  for (const [table, sql] of Object.entries(AUTOMX_TABLES)) {
+  for (const table of Object.keys(AUTOMX_TABLES)) {
     try {
-      answer.tables[table] = parseRows(await exec(container, readCommand(sql)));
+      answer.tables[table] = parseRows(table, await exec(container, readCommand(table)));
     } catch (e) {
       const err = e as { stderr?: string; message?: string };
       const text = (err?.stderr ?? "").trim() || (err?.message ?? String(e)).trim();

modules/mailu/test/automx-rows.test.ts

@@ -2,13 +2,14 @@
 // sqlite3 inside its container. The database here is a real SQLite file made by the test with the tables
 // automx/files/setup-db creates and the rows add-domains writes — with the IMAP server's name empty, the case
 // the live instance is in. The fake container runs each command's statement against that database and prints
-// it the way `sqlite3 -json` does (one JSON array, nothing for no rows). Run against the compiled module.
+// it the way the image's sqlite3 does in list mode (one line per row, cells apart by "|", nothing for no rows); that sqlite3
+// refuses `-json` ("sqlite3: Error: unknown option: -json", captured from novox/mailu.automx_rows, issue 1630). Run against the compiled module.
 
 import { test } from "node:test";
 import assert from "node:assert/strict";
 import { DatabaseSync } from "node:sqlite";
 
-import { readAutomx, readCommand, AUTOMX_DATABASE } from "../dist/automx.js";
+import { readAutomx, readCommand, selectOf, parseRows, AUTOMX_DATABASE } from "../dist/automx.js";
 
 function database(imapName: string): DatabaseSync {
   const db = new DatabaseSync(":memory:");
@@ -32,8 +33,9 @@ function database(imapName: string): DatabaseSync {
 function container(db: DatabaseSync, asked: string[][] = []) {
   return async (_name: string, command: string[]) => {
     asked.push(command);
-    const rows = db.prepare(command[command.length - 1]).all();
+    if (command.includes("-json")) throw Object.assign(new Error("Command failed: docker exec"), { stderr: "sqlite3: Error: unknown option: -json\n" });
-    return rows.length === 0 ? "" : JSON.stringify(rows) + "\n";
+    const rows = db.prepare(command[command.length - 1]).all() as unknown as Record<string, string>[];
+    return rows.length === 0 ? "" : rows.map((r) => Object.values(r).join("|")).join("\n") + "\n";
   };
 }
 
@@ -61,17 +63,18 @@ test("every command opens the file read-only and runs one fixed SELECT", async (
   await readAutomx(container(database("x"), asked), "mailu-automx");
   assert.equal(asked.length, 4);
   for (const c of asked) {
-    assert.deepEqual(c.slice(0, 4), ["sqlite3", "-readonly", "-json", AUTOMX_DATABASE]);
+    assert.deepEqual(c.slice(0, 3), ["sqlite3", "-readonly", AUTOMX_DATABASE]);
-    assert.match(c[4], /^SELECT /);
+    assert.match(c[3], /^SELECT /);
-    assert.equal(c.length, 5);
+    assert.equal(c.length, 4);
+    assert.ok(!c.includes("-json"));
   }
-  assert.deepEqual(readCommand("SELECT 1").length, 5);
+  assert.deepEqual(readCommand("provider"), ["sqlite3", "-readonly", AUTOMX_DATABASE, selectOf("provider")]);
 });
 
 test("a table that cannot be read is said and does not hide the others", async () => {
   const db = database("x");
   const exec = async (name: string, command: string[]) => {
-    if (command[4].includes("FROM domain ")) throw Object.assign(new Error("Command failed: docker exec"), { stderr: "Error: unable to open database file\nmore" });
+    if (command[3].includes("FROM domain ")) throw Object.assign(new Error("Command failed: docker exec"), { stderr: "Error: unable to open database file\nmore" });
     return container(db)(name, command);
   };
   const answer = await readAutomx(exec, "mailu-automx");
@@ -79,3 +82,22 @@ test("a table that cannot be read is said and does not hide the others", async (
   assert.equal(answer.tables.provider.length, 1);
   assert.equal(answer.tables.domain, undefined);
 });
+
+test("a NULL, an empty text, a number and a text holding a bar or a newline are read apart", () => {
+  const hex = (t: string) => Buffer.from(t).toString("hex").toUpperCase();
+  const line = [`integer:${hex("5")}`, `text:${hex("a|b\nc")}`, "text:", "null:"].join("|");
+  const rows = parseRows("domain", line + "\n");
+  assert.deepEqual(rows, [{ id: 5, name: "a|b\nc", provider_id: "", ldapserver_id: null }]);
+});
+
+test("output that is not cells is an unreadable table, not a guess", async () => {
+  const exec = async () => "Error: something else\n";
+  const answer = await readAutomx(exec, "mailu-automx");
+  assert.equal(Object.keys(answer.unreadable).length, 4);
+});
+
+test("the image's refusal of -json is said as the table's reason", async () => {
+  const exec = async (_n: string, command: string[]) => container(database("x"))("", ["sqlite3", "-json", ...command.slice(1)]);
+  const answer = await readAutomx(exec, "mailu-automx");
+  assert.deepEqual(answer.unreadable.domain, "sqlite3: Error: unknown option: -json");
+});