From 32cd92baeb8518218d4d530fc8cc8ee225780a69 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 5 Oct 2026 11:47:59 +0200 Subject: [PATCH] Back up every store: the restic module holds node-backup, the stores contribute their dumps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR 0214 / to-be 43. restic keeps one repository per machine and takes a nightly snapshot per module — 14 daily, 8 weekly, 6 monthly — and restores beside the live data, never over it. postgres, mssql and mongodb contribute a consistent dump; minio, influxdb, the vault, mailu, gitea and nextcloud the directories that hold their data. --- modules/gitea/module.json | 7 + modules/influxdb/module.json | 9 +- modules/mailu/module.json | 7 + modules/mesh-vault/module.json | 7 + modules/minio/module.json | 9 +- modules/mongodb/module.json | 14 +- modules/mssql/module.json | 23 ++- modules/nextcloud/module.json | 9 +- modules/postgres/module.json | 16 +- modules/restic/client.ts | 312 +++++++++++++++++++++++++++++ modules/restic/module.json | 64 ++++++ modules/restic/package.json | 18 ++ modules/restic/test/client.test.ts | 142 +++++++++++++ modules/restic/tools/index.ts | 78 ++++++++ modules/restic/tsconfig.json | 15 ++ 15 files changed, 724 insertions(+), 6 deletions(-) create mode 100644 modules/restic/client.ts create mode 100644 modules/restic/module.json create mode 100644 modules/restic/package.json create mode 100644 modules/restic/test/client.test.ts create mode 100644 modules/restic/tools/index.ts create mode 100644 modules/restic/tsconfig.json diff --git a/modules/gitea/module.json b/modules/gitea/module.json index c329469..16175fc 100644 --- a/modules/gitea/module.json +++ b/modules/gitea/module.json @@ -222,5 +222,12 @@ "failregex": "^.*Failed authentication attempt for .* from (?::\\d+)?\\s*$\n ^.*Invalid user .* from port \\d+\\s*$\n ^.*User \\S+ from not allowed because .*$", "jail": "backend = systemd\njournalmatch = CONTAINER_NAME=gitea\nport = http,https,222\nmaxretry = 3\nfindtime = 1d\nbantime = 1d" } + ], + "contributions": [ + { + "seat": "node-backup", + "kind": "backup", + "content": "path ${dir:data}\n" + } ] } diff --git a/modules/influxdb/module.json b/modules/influxdb/module.json index 0232510..fbc8a67 100644 --- a/modules/influxdb/module.json +++ b/modules/influxdb/module.json @@ -132,5 +132,12 @@ } } ] - } + }, + "contributions": [ + { + "seat": "node-backup", + "kind": "backup", + "content": "path ${dir:data}\npath ${dir:config}\n" + } + ] } diff --git a/modules/mailu/module.json b/modules/mailu/module.json index 4a1cd66..ac5d45c 100644 --- a/modules/mailu/module.json +++ b/modules/mailu/module.json @@ -558,5 +558,12 @@ "failregex": "^.*(?:imap|pop3|submission|managesieve)-login: .*\\(auth failed, \\d+ attempts(?: in \\d+ secs)?\\):.*rip=(?:,|$)", "jail": "backend = systemd\njournalmatch = CONTAINER_NAME=mailu-front\nport = smtp,submission,submissions,imap,imaps,pop3,pop3s\nmaxretry = 3\nfindtime = 1d\nbantime = 1d" } + ], + "contributions": [ + { + "seat": "node-backup", + "kind": "backup", + "content": "path ${dir:data-mail}\npath ${dir:data-dkim}\npath ${dir:data-data}\npath ${dir:data-dav}\npath ${dir:data-webmail}\n" + } ] } diff --git a/modules/mesh-vault/module.json b/modules/mesh-vault/module.json index 31c3092..f68e8ed 100644 --- a/modules/mesh-vault/module.json +++ b/modules/mesh-vault/module.json @@ -79,5 +79,12 @@ "name": "mesh-vault", "scope": "mesh" } + ], + "contributions": [ + { + "seat": "node-backup", + "kind": "backup", + "content": "path ${dir:state}\npath ${dir:ledger}\npath ${dir:root}\n" + } ] } diff --git a/modules/minio/module.json b/modules/minio/module.json index 6a0bc85..ac6b7aa 100644 --- a/modules/minio/module.json +++ b/modules/minio/module.json @@ -151,5 +151,12 @@ } } ] - } + }, + "contributions": [ + { + "seat": "node-backup", + "kind": "backup", + "content": "path ${dir:data}\n" + } + ] } diff --git a/modules/mongodb/module.json b/modules/mongodb/module.json index 1f70756..6454914 100644 --- a/modules/mongodb/module.json +++ b/modules/mongodb/module.json @@ -58,6 +58,11 @@ "type": "directory", "mode": "0700" }, + { + "id": "dumps", + "type": "directory", + "mode": "0700" + }, { "id": "net", "type": "network", @@ -113,5 +118,12 @@ } } ] - } + }, + "contributions": [ + { + "seat": "node-backup", + "kind": "backup", + "content": "run docker exec mongodb-server sh -c 'printf \"password: %s\\n\" \"$(cat /run/secrets/root)\" > /tmp/.backup.yaml && mongodump --quiet --config /tmp/.backup.yaml --username root --authenticationDatabase admin --archive; s=$?; rm -f /tmp/.backup.yaml; exit $s' > ${dir:dumps}/all.archive.partial && mv ${dir:dumps}/all.archive.partial ${dir:dumps}/all.archive\npath ${dir:dumps}\n" + } + ] } diff --git a/modules/mssql/module.json b/modules/mssql/module.json index 28802a7..5a44231 100644 --- a/modules/mssql/module.json +++ b/modules/mssql/module.json @@ -65,6 +65,13 @@ "mode": "0700", "owner": "10001:0" }, + { + "id": "dumps", + "type": "directory", + "path": "${dir:data}/backup", + "mode": "0700", + "owner": "10001:0" + }, { "id": "net", "type": "network", @@ -86,6 +93,13 @@ "${dir:data}:/var/opt/mssql" ], "secrets-in-environment": "the image documents only MSSQL_SA_PASSWORD, no _FILE and no configuration field; not convertible without a wrapper entrypoint" + }, + { + "id": "backup-sql", + "type": "file", + "path": "${dir:state}/backup.sql", + "mode": "0600", + "content": "SET NOCOUNT ON;\nDECLARE @n sysname, @s nvarchar(max);\nDECLARE c CURSOR LOCAL FAST_FORWARD FOR\n SELECT name FROM sys.databases WHERE database_id > 4 AND state = 0 AND source_database_id IS NULL;\nOPEN c;\nFETCH NEXT FROM c INTO @n;\nWHILE @@FETCH_STATUS = 0\nBEGIN\n SET @s = N'BACKUP DATABASE ' + QUOTENAME(@n) + N' TO DISK = N''/var/opt/mssql/backup/' + REPLACE(@n, N'''', N'''''') + N'.bak'' WITH INIT, COPY_ONLY, CHECKSUM';\n EXEC (@s);\n FETCH NEXT FROM c INTO @n;\nEND\nCLOSE c;\nDEALLOCATE c;\n" } ], "build": { @@ -112,5 +126,12 @@ } } ] - } + }, + "contributions": [ + { + "seat": "node-backup", + "kind": "backup", + "content": "run { cat ${dir:state}/sa.secret; echo; cat ${dir:state}/backup.sql; } | docker exec -i mssql sh -c 'read -r p; SQLCMDPASSWORD=\"$p\" exec /opt/mssql-tools18/bin/sqlcmd -C -b -S localhost -U sa -i /dev/stdin'\npath ${dir:dumps}\n" + } + ] } diff --git a/modules/nextcloud/module.json b/modules/nextcloud/module.json index 39fac38..fb177b4 100644 --- a/modules/nextcloud/module.json +++ b/modules/nextcloud/module.json @@ -117,5 +117,12 @@ } } ] - } + }, + "contributions": [ + { + "seat": "node-backup", + "kind": "backup", + "content": "path ${dir:html}\n" + } + ] } diff --git a/modules/postgres/module.json b/modules/postgres/module.json index f25d84d..dc1a256 100644 --- a/modules/postgres/module.json +++ b/modules/postgres/module.json @@ -74,6 +74,13 @@ "mode": "0700", "owner": "999:70" }, + { + "id": "dumps", + "type": "directory", + "path": "${dir:store-data}/dumps", + "mode": "0700", + "owner": "999:70" + }, { "id": "server", "type": "container", @@ -121,5 +128,12 @@ } } ] - } + }, + "contributions": [ + { + "seat": "node-backup", + "kind": "backup", + "content": "run docker exec -u postgres postgres sh -c 'cd /var/lib/postgresql/data/dumps && for db in $(psql -Atc \"select datname from pg_database where oid >= 16384 order by 1\"); do pg_dump -Fc -f \"$db.dump.partial\" \"$db\" && mv \"$db.dump.partial\" \"$db.dump\" || exit 1; done'\npath ${dir:dumps}\n" + } + ] } diff --git a/modules/restic/client.ts b/modules/restic/client.ts new file mode 100644 index 0000000..884f2af --- /dev/null +++ b/modules/restic/client.ts @@ -0,0 +1,312 @@ +// restic's own code, in the module (novox/hq ADR 0039): the machine's backups (ADR 0214, to-be 43). +// +// The mesh composes what to back up: every module on the machine contributes `backup` lines to the +// node-backup seat, and the mesh writes them, each module's under a `# ` line and with its +// directories already filled, into one file this module reads. Two kinds of line: +// +// run run as root before the module's snapshot — a consistent dump of a store +// path a directory the module's snapshot keeps +// +// Each module gets one snapshot a night, tagged with its name, so a module is listed, kept and +// restored on its own. Everything lands in one repository on the machine — deduplicated, so every +// night is a complete restore point and only what changed costs space — and is thinned to 14 daily, +// 8 weekly and 6 monthly. Against mistakes, not disasters: nothing leaves the machine. +// +// Root's: the dumps read every store and the repository holds every module's data, and the runtime +// loading this bundle runs as the operator's account, so restic and the run lines go through sudo +// without a prompt where the account is not root — fail2ban's way (ADR 0175 §4). + +import { execFile } from "node:child_process"; +import { existsSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { promisify } from "node:util"; + +const execFileP = promisify(execFile); + +/** A command runner, so the backups can be tested without restic or a store. */ +export type Runner = (cmd: string, args: string[]) => Promise; + +/** The command as it is run: as given when this process is root, else through sudo without a prompt. */ +export function escalated(cmd: string, args: string[], uid: number | undefined = process.getuid?.()): [string, string[]] { + if (uid === 0) return [cmd, args]; + return ["sudo", ["-n", cmd, ...args]]; +} + +export const execRunner: Runner = async (cmd, args) => { + const [program, argv] = escalated(cmd, args); + try { + // A night's dump of a large store takes a while; six hours is a dump that will not finish. + const { stdout } = await execFileP(program, argv, { maxBuffer: 256 * 1024 * 1024, timeout: 6 * 3600 * 1000 }); + return stdout; + } catch (err) { + const e = err as { code?: string | number; stderr?: string; stdout?: string; message?: string }; + const said = `${e.stderr ?? ""}`.trim() || `${e.stdout ?? ""}`.trim(); + if (program === "sudo" && /^sudo:/m.test(said)) { + throw new Error(`${cmd} needs root and the runtime's account may not run it without a prompt: ${said}`); + } + const lines = said.split("\n").map((l) => l.trim()).filter(Boolean); + throw new Error(lines.length ? lines.slice(-3).join(" / ") : (e.message ?? `${cmd} failed`)); + } +}; + +/** What one module declared. */ +export interface Declared { + module: string; + runs: string[]; + paths: string[]; +} + +/** The composed file, read into each module's declaration, in the order the mesh wrote them. A line + * before any module, a comment that is not a module's name, or a blank, is nothing. */ +export function parseDeclared(text: string): Declared[] { + const out: Declared[] = []; + let current: Declared | undefined; + for (const raw of text.split("\n")) { + const line = raw.trim(); + if (line === "") continue; + const header = /^#\s*([a-z0-9][a-z0-9-]*)$/.exec(line); + if (header) { + current = { module: header[1], runs: [], paths: [] }; + out.push(current); + continue; + } + if (line.startsWith("#") || !current) continue; + const [kind, ...rest] = line.split(/\s+/); + const value = line.slice(kind.length).trim(); + if (kind === "run" && value) current.runs.push(value); + else if (kind === "path" && rest.length === 1 && value.startsWith("/")) current.paths.push(value); + else throw new Error(`${current.module} contributes a backup line this holder does not read: ${line}`); + } + return out.filter((d) => d.runs.length > 0 || d.paths.length > 0); +} + +/** One restore point, as restic lists it. */ +export interface Snapshot { + id: string; + short_id: string; + time: string; + paths: string[]; + tags?: string[]; + hostname: string; +} + +/** How one module's last night went. */ +export interface Night { + ok: boolean; + at: string; + snapshot?: string; + error?: string; +} + +export const KEEP = { daily: 14, weekly: 8, monthly: 6 }; + +export function tagOf(module: string): string { + return `module=${module}`; +} + +export interface Where { + declared: string; + repository: string; + passwordFile: string; + state: string; +} + +export function whereFromEnv(env: NodeJS.ProcessEnv = process.env): Where { + const need = (k: string) => { + const v = env[k]; + if (!v) throw new Error(`${k} is not set; the mesh gives it to this module's tools`); + return v; + }; + return { + declared: need("MESH_BACKUP_DECLARED"), + repository: need("MESH_BACKUP_REPOSITORY"), + passwordFile: need("MESH_BACKUP_PASSWORD_FILE"), + state: need("MESH_BACKUP_STATE"), + }; +} + +export class Backups { + private busy: Promise = Promise.resolve(); + readonly where: Where; + private run: Runner; + private readonly now: () => Date; + private readonly say: (line: string) => void; + + constructor( + where: Where, + run: Runner = execRunner, + now: () => Date = () => new Date(), + say: (line: string) => void = (l) => console.error(`[restic] ${l}`), + ) { + this.where = where; + this.run = run; + this.now = now; + this.say = say; + } + + private restic(args: string[]): Promise { + return this.run("restic", ["--repo", this.where.repository, "--password-file", this.where.passwordFile, "--no-cache", ...args]); + } + + /** One thing at a time: two nights, or a night and a restore, never share a dump. */ + private serial(work: () => Promise): Promise { + const next = this.busy.then(work, work); + this.busy = next.catch(() => undefined); + return next; + } + + declared(): Declared[] { + return parseDeclared(readFileSync(this.where.declared, "utf8")); + } + + private nightsFile(): string { + return join(this.where.state, "nights.json"); + } + + nights(): Record { + try { + return JSON.parse(readFileSync(this.nightsFile(), "utf8")) as Record; + } catch { + return {}; + } + } + + private record(module: string, night: Night): void { + const all = this.nights(); + all[module] = night; + writeFileSync(this.nightsFile(), JSON.stringify(all, null, 2) + "\n", { mode: 0o600 }); + } + + /** The repository, made the first time. A repository that exists and cannot be opened is said, + * never replaced: replacing it would discard every restore point to fix a password. */ + async ensureRepository(): Promise { + try { + await this.restic(["cat", "config"]); + } catch (err) { + const why = String((err as Error).message); + if (!/does not exist|unable to open config file|Is there a repository at the following location/i.test(why)) { + throw new Error(`the repository at ${this.where.repository} cannot be opened, and is left as it is: ${why}`); + } + this.say(`no repository at ${this.where.repository}; making one`); + await this.restic(["init"]); + } + } + + /** One module's night: its run lines, then one snapshot of its paths. A failure is that module's. */ + private async one(d: Declared): Promise { + const at = this.now().toISOString(); + try { + for (const command of d.runs) { + await this.run("sh", ["-c", command]); + } + const missing = d.paths.filter((p) => !existsSync(p)); + if (missing.length > 0) throw new Error(`${missing.join(", ")} does not exist`); + if (d.paths.length === 0) throw new Error("it runs a dump and names no directory to keep it from"); + const out = await this.restic(["backup", "--json", "--tag", tagOf(d.module), ...d.paths]); + const summary = out + .split("\n") + .map((l) => { try { return JSON.parse(l) as { message_type?: string; snapshot_id?: string }; } catch { return {}; } }) + .find((m) => m.message_type === "summary"); + const night: Night = { ok: true, at, snapshot: summary?.snapshot_id?.slice(0, 8) }; + this.say(`${d.module}: backed up (${night.snapshot ?? "no snapshot id reported"})`); + return night; + } catch (err) { + const night: Night = { ok: false, at, error: String((err as Error).message) }; + this.say(`${d.module}: NOT backed up: ${night.error}`); + return night; + } + } + + /** A night: every module, or one, then the rotation. Returns each module's outcome. */ + backUp(only?: string): Promise> { + return this.serial(async () => { + await this.ensureRepository(); + const all = this.declared(); + const chosen = only ? all.filter((d) => d.module === only) : all; + if (only && chosen.length === 0) { + throw new Error(`${only} declares nothing to back up on this machine; it backs up ${all.map((d) => d.module).join(", ") || "nothing"}`); + } + const outcome: Record = {}; + for (const d of chosen) { + outcome[d.module] = await this.one(d); + this.record(d.module, outcome[d.module]); + } + await this.restic([ + "forget", "--prune", "--group-by", "host,tags", + "--keep-daily", String(KEEP.daily), "--keep-weekly", String(KEEP.weekly), "--keep-monthly", String(KEEP.monthly), + ]).catch((err) => this.say(`thinning the restore points failed, and every one is kept: ${(err as Error).message}`)); + return outcome; + }); + } + + async snapshots(module?: string): Promise { + const args = ["snapshots", "--json"]; + if (module) args.push("--tag", tagOf(module)); + return JSON.parse((await this.restic(args)) || "[]") as Snapshot[]; + } + + /** What is backed up here: each module, what it declared, its last night and its restore points. */ + async backedUp(module?: string) { + const nights = this.nights(); + const snaps = await this.snapshots(module).catch(() => [] as Snapshot[]); + return this.declared() + .filter((d) => !module || d.module === module) + .map((d) => { + const mine = snaps.filter((s) => (s.tags ?? []).includes(tagOf(d.module))); + return { + module: d.module, + runs: d.runs.length, + paths: d.paths, + lastNight: nights[d.module] ?? null, + restorePoints: mine.length, + newest: mine.length ? { snapshot: mine[mine.length - 1].short_id, at: mine[mine.length - 1].time } : null, + }; + }); + } + + /** A module's data from a restore point, BESIDE the live data: each directory as + * .restored-. A target that already exists is refused, never overwritten. */ + restore(module: string, snapshot?: string, path?: string) { + return this.serial(async () => { + const mine = await this.snapshots(module); + if (mine.length === 0) throw new Error(`${module} has no restore point on this machine`); + const chosen = snapshot ? mine.find((s) => s.id.startsWith(snapshot) || s.short_id === snapshot) : mine[mine.length - 1]; + if (!chosen) { + throw new Error(`${module} has no restore point ${snapshot}; it has ${mine.map((s) => `${s.short_id} (${s.time})`).join(", ")}`); + } + const paths = path ? chosen.paths.filter((p) => p === path) : chosen.paths; + if (paths.length === 0) throw new Error(`restore point ${chosen.short_id} of ${module} holds ${chosen.paths.join(", ")}, not ${path}`); + const stamp = this.now().toISOString().replace(/[-:]/g, "").replace("T", "-").slice(0, 15); + const restored: string[] = []; + for (const p of paths) { + const target = `${p}.restored-${stamp}`; + if (existsSync(target)) throw new Error(`${target} already exists; nothing is restored over anything`); + await this.restic(["restore", `${chosen.id}:${p}`, "--target", target]); + restored.push(target); + this.say(`${module}: restored ${p} from ${chosen.short_id} to ${target}`); + } + return { module, from: { snapshot: chosen.short_id, at: chosen.time }, restored, live: "untouched — swapping it in is a person's act" }; + }); + } + + /** The weekly look at the repository's own integrity, with a sample of the data read back. */ + check(): Promise { + return this.serial(() => this.restic(["check", "--read-data-subset", "5%"])); + } +} + +/** When the next night is due: the given hour, local time, today if it is still ahead, else tomorrow. */ +export function nextNight(now: Date, hour: number): Date { + const next = new Date(now); + next.setHours(hour, 0, 0, 0); + if (next.getTime() <= now.getTime()) next.setDate(next.getDate() + 1); + return next; +} + +/** Whether a night was missed: the newest good night of any module is older than a day and a bit — + * the machine was off, or this module was not running, at the hour. */ +export function missedANight(nights: Record, now: Date): boolean { + const good = Object.values(nights).filter((n) => n.ok).map((n) => Date.parse(n.at)); + if (good.length === 0) return true; + return now.getTime() - Math.max(...good) > 26 * 3600 * 1000; +} diff --git a/modules/restic/module.json b/modules/restic/module.json new file mode 100644 index 0000000..f76a457 --- /dev/null +++ b/modules/restic/module.json @@ -0,0 +1,64 @@ +{ + "module": "restic", + "version": "1", + "claims": [ + { + "name": "node-backup", + "scope": "node", + "serves": [ + "backed-up", + "now", + "restore" + ] + } + ], + "own-secrets": { + "repository": "${dir:state}/repository.secret" + }, + "resources": [ + { + "id": "state", + "type": "directory", + "mode": "0700", + "place": "." + }, + { + "id": "repository", + "type": "directory", + "mode": "0700" + }, + { + "id": "declared", + "type": "file", + "path": "${dir:state}/backups.conf", + "mode": "0600", + "content": "# What the modules on this machine back up, composed by the mesh (novox/hq to-be 43). Do not edit.\n${contribution:node-backup:backup}" + }, + { + "id": "tool", + "type": "package", + "package": "restic" + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "typescript", + "entrypoints": [ + "tools/index.js" + ], + "loads": [ + "tools/index.js" + ], + "env": { + "MESH_BACKUP_DECLARED": "${dir:state}/backups.conf", + "MESH_BACKUP_REPOSITORY": "${dir:repository}", + "MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret", + "MESH_BACKUP_STATE": "${dir:state}" + } + } + ] + } +} diff --git a/modules/restic/package.json b/modules/restic/package.json new file mode 100644 index 0000000..bdcffc0 --- /dev/null +++ b/modules/restic/package.json @@ -0,0 +1,18 @@ +{ + "name": "@novox/module-restic", + "version": "0.1.0", + "description": "restic \u2014 the machine's backups: holds the node-backup seat, takes a nightly restore point of what every module on the machine declares, keeps 14 daily, 8 weekly and 6 monthly, and restores beside the live data (novox/hq ADR 0214, to-be 43).", + "type": "module", + "private": true, + "dependencies": { + "@novox/mesh-sdk": "^0.1.1" + }, + "devDependencies": { + "@types/node": "^22.0.0", + "typescript": "^5.6.0" + }, + "scripts": { + "build": "tsc client.ts tools/index.ts --module NodeNext --moduleResolution NodeNext --target ES2022 --rootDir . --outDir dist", + "test": "node --test --experimental-strip-types 'test/*.test.ts'" + } +} diff --git a/modules/restic/test/client.test.ts b/modules/restic/test/client.test.ts new file mode 100644 index 0000000..599dd2f --- /dev/null +++ b/modules/restic/test/client.test.ts @@ -0,0 +1,142 @@ +// The machine's backups over a fake restic and fake stores (novox/hq ADR 0214, to-be 43), and — where +// restic is installed — over the real one, on throwaway directories. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Backups, escalated, missedANight, nextNight, parseDeclared, type Runner } from "../client.ts"; + +const COMPOSED = + "# What the modules on this machine back up, composed by the mesh. Do not edit.\n" + + "# postgres\nrun docker exec -u postgres postgres sh -c 'pg-dump-all'\npath /var/lib/mesh-store/dumps\n" + + "# mailu\npath /var/lib/mailu/data-mail\npath /var/lib/mailu/data-dkim\n"; + +function place(declared: string) { + const dir = mkdtempSync(join(tmpdir(), "restic-test-")); + writeFileSync(join(dir, "backups.conf"), declared); + writeFileSync(join(dir, "pw"), "secret\n"); + return { declared: join(dir, "backups.conf"), repository: join(dir, "repo"), passwordFile: join(dir, "pw"), state: dir }; +} + +test("the composed file is read into each module's runs and paths, the header comment being nothing", () => { + assert.deepEqual(parseDeclared(COMPOSED), [ + { module: "postgres", runs: ["docker exec -u postgres postgres sh -c 'pg-dump-all'"], paths: ["/var/lib/mesh-store/dumps"] }, + { module: "mailu", runs: [], paths: ["/var/lib/mailu/data-mail", "/var/lib/mailu/data-dkim"] }, + ]); +}); + +test("a line this holder does not read is refused, naming the module, rather than skipped", () => { + assert.throws(() => parseDeclared("# pg\ncopy /x\n"), /pg contributes a backup line this holder does not read: copy \/x/); + assert.throws(() => parseDeclared("# pg\npath relative/dir\n"), /pg contributes/); +}); + +test("restic and the dumps run through sudo where the account is not root", () => { + assert.deepEqual(escalated("restic", ["snapshots"], 1000), ["sudo", ["-n", "restic", "snapshots"]]); + assert.deepEqual(escalated("restic", ["snapshots"], 0), ["restic", ["snapshots"]]); +}); + +test("a night runs each module's dump before its snapshot, and one failing module fails only itself", async () => { + const where = place("# pg\nrun dump-it\npath /\n# broken\nrun fail-it\npath /\n# mail\npath /\n"); + const calls: string[] = []; + const run: Runner = async (cmd, args) => { + const line = cmd === "restic" ? `restic ${args.slice(5).join(" ")}` : `${cmd} ${args.join(" ")}`; + calls.push(line); + if (line === "sh -c fail-it") throw new Error("the dump failed"); + if (line.startsWith("restic backup")) return '{"message_type":"status"}\n{"message_type":"summary","snapshot_id":"abcdef0123456789"}\n'; + return ""; + }; + const outcome = await new Backups(where, run, () => new Date("2026-10-06T03:00:00Z"), () => {}).backUp(); + assert.equal(outcome.pg.ok, true); + assert.equal(outcome.pg.snapshot, "abcdef01"); + assert.equal(outcome.broken.ok, false); + assert.match(outcome.broken.error ?? "", /the dump failed/); + assert.equal(outcome.mail.ok, true); + assert.deepEqual(calls, [ + "restic cat config", + "sh -c dump-it", + "restic backup --json --tag module=pg /", + "sh -c fail-it", + "restic backup --json --tag module=mail /", + "restic forget --prune --group-by host,tags --keep-daily 14 --keep-weekly 8 --keep-monthly 6", + ]); + // Recorded, so `backed-up` and the missed-night check read it. + const nights = JSON.parse(readFileSync(join(where.state, "nights.json"), "utf8")); + assert.equal(nights.broken.ok, false); + assert.equal(nights.mail.ok, true); +}); + +test("a repository that exists and will not open is never replaced", async () => { + const where = place("# pg\npath /\n"); + const calls: string[] = []; + const run: Runner = async (cmd, args) => { + calls.push(args.slice(5).join(" ")); + if (args.includes("cat")) throw new Error("Fatal: wrong password or no key found"); + return ""; + }; + await assert.rejects(new Backups(where, run, undefined, () => {}).backUp(), /cannot be opened, and is left as it is/); + assert.ok(!calls.includes("init"), "it made a new repository over one it could not open"); +}); + +test("a declared directory that does not exist fails that module's night", async () => { + const where = place("# pg\npath /nowhere/at/all\n"); + const run: Runner = async () => ""; + const outcome = await new Backups(where, run, undefined, () => {}).backUp(); + assert.equal(outcome.pg.ok, false); + assert.match(outcome.pg.error ?? "", /\/nowhere\/at\/all does not exist/); +}); + +test("a night is due at the hour today while it is ahead, else tomorrow; a missed one is noticed", () => { + const morning = new Date(2026, 9, 6, 1, 30); + assert.equal(nextNight(morning, 3).getTime(), new Date(2026, 9, 6, 3, 0).getTime()); + const afternoon = new Date(2026, 9, 6, 15, 0); + assert.equal(nextNight(afternoon, 3).getTime(), new Date(2026, 9, 7, 3, 0).getTime()); + assert.equal(missedANight({}, afternoon), true); + assert.equal(missedANight({ pg: { ok: true, at: new Date(2026, 9, 6, 3, 5).toISOString() } }, afternoon), false); + assert.equal(missedANight({ pg: { ok: true, at: new Date(2026, 9, 4, 3, 5).toISOString() } }, afternoon), true); + assert.equal(missedANight({ pg: { ok: false, at: new Date(2026, 9, 6, 3, 5).toISOString() } }, afternoon), true); +}); + +// The real thing, where restic is installed: a dump, a snapshot, a mistake, and a restore beside. +const hasRestic = (() => { + try { + execFileSync("restic", ["version"], { stdio: "ignore" }); + return true; + } catch { + return false; + } +})(); + +test("with the real restic: a mistake is undone by a restore beside the live data", { skip: !hasRestic && "restic is not installed" }, async () => { + const root = mkdtempSync(join(tmpdir(), "restic-real-")); + const store = join(root, "store"); + const dumps = join(root, "dumps"); + mkdirSync(store); + mkdirSync(dumps); + writeFileSync(join(store, "mailbox"), "the only copy of a letter\n"); + const where = place(`# mail\npath ${store}\n# pg\nrun echo 'every row' > ${dumps}/all.dump\npath ${dumps}\n`); + const backups = new Backups(where, undefined, () => new Date("2026-10-06T03:00:00Z"), () => {}); + // escalated() would sudo; the test runs as whoever it is, against its own repository. + (backups as unknown as { run: Runner }).run = async (cmd, args) => execFileSync(cmd, args, { encoding: "utf8" }); + + const night = await backups.backUp(); + assert.equal(night.mail.ok, true, night.mail.error); + assert.equal(night.pg.ok, true, night.pg.error); + assert.equal(readFileSync(join(dumps, "all.dump"), "utf8"), "every row\n"); + + // The mistake. + rmSync(join(store, "mailbox")); + + const listed = await backups.backedUp(); + assert.deepEqual(listed.map((m) => [m.module, m.restorePoints]), [["mail", 1], ["pg", 1]]); + + const restored = await backups.restore("mail"); + assert.equal(restored.restored.length, 1); + assert.match(restored.restored[0], /store\.restored-20261006-030000$/); + assert.equal(readFileSync(join(restored.restored[0], "mailbox"), "utf8"), "the only copy of a letter\n"); + assert.ok(!existsSync(join(store, "mailbox")), "the restore wrote into the live directory"); + + // Never over anything: the same restore again finds its target taken. + await assert.rejects(backups.restore("mail"), /already exists; nothing is restored over anything/); +}); diff --git a/modules/restic/tools/index.ts b/modules/restic/tools/index.ts new file mode 100644 index 0000000..95f15c1 --- /dev/null +++ b/modules/restic/tools/index.ts @@ -0,0 +1,78 @@ +// The machine's backups: the node-backup seat's three verbs — what is backed up, take one now, +// restore beside the live data — and the night that runs without anyone asking (novox/hq ADR 0214, +// to-be 43). What is backed up is composed by the mesh from the modules the machine runs; this code +// only runs it. + +import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools"; +import { Backups, missedANight, nextNight, whereFromEnv } from "../client.js"; + +export function getSeatVerbs(backups: Backups): ToolDefinition[] { + return [ + { + name: "backed-up", + description: + "What this machine backs up: each module, what it declared, its last good night, how many restore points are kept.", + input: { module: { type: "string", description: "one module (optional)" } }, + run: async (args) => backups.backedUp(args.module ? String(args.module) : undefined), + }, + { + name: "now", + description: + "Take a backup now, of one module or of every module on this machine — before a migration, a retirement or anything else that could go wrong. Answers when it has started; `backed-up` says how it went.", + input: { module: { type: "string", description: "one module (optional)" } }, + run: async (args) => { + const only = args.module ? String(args.module) : undefined; + // A night of a large store outlasts any call; it is started, and its outcome recorded. + backups.backUp(only).catch((err) => console.error(`[restic] a backup asked for now failed: ${(err as Error).message}`)); + return { started: only ?? "every module on this machine", follow: "backed-up" }; + }, + }, + { + name: "restore", + description: + "Restore one module's data from a restore point BESIDE the live data, never over it: each directory as .restored-. Swapping it in is a person's act.", + input: { + module: { type: "string", description: "the module" }, + snapshot: { type: "string", description: "the restore point (the newest when omitted)" }, + path: { type: "string", description: "one of the module's directories (all of them when omitted)" }, + }, + run: async (args) => + backups.restore(String(args.module ?? ""), args.snapshot ? String(args.snapshot) : undefined, args.path ? String(args.path) : undefined), + }, + ]; +} + +const backups = new Backups(whereFromEnv()); +registerModuleTools("node-backup", () => getSeatVerbs(backups)); + +// The night. At the hour, every module; and at start, if a night was missed — the machine was off +// or this module was not running at the hour — one now rather than a day later. +const HOUR = Number(process.env.MESH_BACKUP_HOUR ?? "3"); + +async function night(): Promise { + try { + const outcome = await backups.backUp(); + const failed = Object.entries(outcome).filter(([, n]) => !n.ok).map(([m]) => m); + if (failed.length > 0) console.error(`[restic] the night left ${failed.join(", ")} without a backup`); + // Sundays, the repository's own integrity with a sample of the data read back. + if (new Date().getDay() === 0) { + await backups.check().then( + () => console.error("[restic] the repository checks out"), + (err) => console.error(`[restic] the repository does NOT check out: ${(err as Error).message}`), + ); + } + } catch (err) { + console.error(`[restic] the night did not run: ${(err as Error).message}`); + } +} + +function schedule(): void { + const at = nextNight(new Date(), HOUR); + setTimeout(() => void night().finally(schedule), at.getTime() - Date.now()); +} + +if (missedANight(backups.nights(), new Date())) { + // Not at once: a machine just started has its stores still coming up. + setTimeout(() => void night(), 10 * 60 * 1000); +} +schedule(); diff --git a/modules/restic/tsconfig.json b/modules/restic/tsconfig.json new file mode 100644 index 0000000..1f1b70a --- /dev/null +++ b/modules/restic/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "noEmit": true + }, + "include": [ + "client.ts", + "tools/index.ts" + ] +}