From 32cd92baeb8518218d4d530fc8cc8ee225780a69 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 5 Oct 2026 11:47:59 +0200 Subject: [PATCH 1/2] Back up every store: the restic module holds node-backup, the stores contribute their dumps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR 0214 / to-be 43. restic keeps one repository per machine and takes a nightly snapshot per module — 14 daily, 8 weekly, 6 monthly — and restores beside the live data, never over it. postgres, mssql and mongodb contribute a consistent dump; minio, influxdb, the vault, mailu, gitea and nextcloud the directories that hold their data. --- modules/gitea/module.json | 7 + modules/influxdb/module.json | 9 +- modules/mailu/module.json | 7 + modules/mesh-vault/module.json | 7 + modules/minio/module.json | 9 +- modules/mongodb/module.json | 14 +- modules/mssql/module.json | 23 ++- modules/nextcloud/module.json | 9 +- modules/postgres/module.json | 16 +- modules/restic/client.ts | 312 +++++++++++++++++++++++++++++ modules/restic/module.json | 64 ++++++ modules/restic/package.json | 18 ++ modules/restic/test/client.test.ts | 142 +++++++++++++ modules/restic/tools/index.ts | 78 ++++++++ modules/restic/tsconfig.json | 15 ++ 15 files changed, 724 insertions(+), 6 deletions(-) create mode 100644 modules/restic/client.ts create mode 100644 modules/restic/module.json create mode 100644 modules/restic/package.json create mode 100644 modules/restic/test/client.test.ts create mode 100644 modules/restic/tools/index.ts create mode 100644 modules/restic/tsconfig.json diff --git a/modules/gitea/module.json b/modules/gitea/module.json index c329469..16175fc 100644 --- a/modules/gitea/module.json +++ b/modules/gitea/module.json @@ -222,5 +222,12 @@ "failregex": "^.*Failed authentication attempt for .* from (?::\\d+)?\\s*$\n ^.*Invalid user .* from port \\d+\\s*$\n ^.*User \\S+ from not allowed because .*$", "jail": "backend = systemd\njournalmatch = CONTAINER_NAME=gitea\nport = http,https,222\nmaxretry = 3\nfindtime = 1d\nbantime = 1d" } + ], + "contributions": [ + { + "seat": "node-backup", + "kind": "backup", + "content": "path ${dir:data}\n" + } ] } diff --git a/modules/influxdb/module.json b/modules/influxdb/module.json index 0232510..fbc8a67 100644 --- a/modules/influxdb/module.json +++ b/modules/influxdb/module.json @@ -132,5 +132,12 @@ } } ] - } + }, + "contributions": [ + { + "seat": "node-backup", + "kind": "backup", + "content": "path ${dir:data}\npath ${dir:config}\n" + } + ] } diff --git a/modules/mailu/module.json b/modules/mailu/module.json index 4a1cd66..ac5d45c 100644 --- a/modules/mailu/module.json +++ b/modules/mailu/module.json @@ -558,5 +558,12 @@ "failregex": "^.*(?:imap|pop3|submission|managesieve)-login: .*\\(auth failed, \\d+ attempts(?: in \\d+ secs)?\\):.*rip=(?:,|$)", "jail": "backend = systemd\njournalmatch = CONTAINER_NAME=mailu-front\nport = smtp,submission,submissions,imap,imaps,pop3,pop3s\nmaxretry = 3\nfindtime = 1d\nbantime = 1d" } + ], + "contributions": [ + { + "seat": "node-backup", + "kind": "backup", + "content": "path ${dir:data-mail}\npath ${dir:data-dkim}\npath ${dir:data-data}\npath ${dir:data-dav}\npath ${dir:data-webmail}\n" + } ] } diff --git a/modules/mesh-vault/module.json b/modules/mesh-vault/module.json index 31c3092..f68e8ed 100644 --- a/modules/mesh-vault/module.json +++ b/modules/mesh-vault/module.json @@ -79,5 +79,12 @@ "name": "mesh-vault", "scope": "mesh" } + ], + "contributions": [ + { + "seat": "node-backup", + "kind": "backup", + "content": "path ${dir:state}\npath ${dir:ledger}\npath ${dir:root}\n" + } ] } diff --git a/modules/minio/module.json b/modules/minio/module.json index 6a0bc85..ac6b7aa 100644 --- a/modules/minio/module.json +++ b/modules/minio/module.json @@ -151,5 +151,12 @@ } } ] - } + }, + "contributions": [ + { + "seat": "node-backup", + "kind": "backup", + "content": "path ${dir:data}\n" + } + ] } diff --git a/modules/mongodb/module.json b/modules/mongodb/module.json index 1f70756..6454914 100644 --- a/modules/mongodb/module.json +++ b/modules/mongodb/module.json @@ -58,6 +58,11 @@ "type": "directory", "mode": "0700" }, + { + "id": "dumps", + "type": "directory", + "mode": "0700" + }, { "id": "net", "type": "network", @@ -113,5 +118,12 @@ } } ] - } + }, + "contributions": [ + { + "seat": "node-backup", + "kind": "backup", + "content": "run docker exec mongodb-server sh -c 'printf \"password: %s\\n\" \"$(cat /run/secrets/root)\" > /tmp/.backup.yaml && mongodump --quiet --config /tmp/.backup.yaml --username root --authenticationDatabase admin --archive; s=$?; rm -f /tmp/.backup.yaml; exit $s' > ${dir:dumps}/all.archive.partial && mv ${dir:dumps}/all.archive.partial ${dir:dumps}/all.archive\npath ${dir:dumps}\n" + } + ] } diff --git a/modules/mssql/module.json b/modules/mssql/module.json index 28802a7..5a44231 100644 --- a/modules/mssql/module.json +++ b/modules/mssql/module.json @@ -65,6 +65,13 @@ "mode": "0700", "owner": "10001:0" }, + { + "id": "dumps", + "type": "directory", + "path": "${dir:data}/backup", + "mode": "0700", + "owner": "10001:0" + }, { "id": "net", "type": "network", @@ -86,6 +93,13 @@ "${dir:data}:/var/opt/mssql" ], "secrets-in-environment": "the image documents only MSSQL_SA_PASSWORD, no _FILE and no configuration field; not convertible without a wrapper entrypoint" + }, + { + "id": "backup-sql", + "type": "file", + "path": "${dir:state}/backup.sql", + "mode": "0600", + "content": "SET NOCOUNT ON;\nDECLARE @n sysname, @s nvarchar(max);\nDECLARE c CURSOR LOCAL FAST_FORWARD FOR\n SELECT name FROM sys.databases WHERE database_id > 4 AND state = 0 AND source_database_id IS NULL;\nOPEN c;\nFETCH NEXT FROM c INTO @n;\nWHILE @@FETCH_STATUS = 0\nBEGIN\n SET @s = N'BACKUP DATABASE ' + QUOTENAME(@n) + N' TO DISK = N''/var/opt/mssql/backup/' + REPLACE(@n, N'''', N'''''') + N'.bak'' WITH INIT, COPY_ONLY, CHECKSUM';\n EXEC (@s);\n FETCH NEXT FROM c INTO @n;\nEND\nCLOSE c;\nDEALLOCATE c;\n" } ], "build": { @@ -112,5 +126,12 @@ } } ] - } + }, + "contributions": [ + { + "seat": "node-backup", + "kind": "backup", + "content": "run { cat ${dir:state}/sa.secret; echo; cat ${dir:state}/backup.sql; } | docker exec -i mssql sh -c 'read -r p; SQLCMDPASSWORD=\"$p\" exec /opt/mssql-tools18/bin/sqlcmd -C -b -S localhost -U sa -i /dev/stdin'\npath ${dir:dumps}\n" + } + ] } diff --git a/modules/nextcloud/module.json b/modules/nextcloud/module.json index 39fac38..fb177b4 100644 --- a/modules/nextcloud/module.json +++ b/modules/nextcloud/module.json @@ -117,5 +117,12 @@ } } ] - } + }, + "contributions": [ + { + "seat": "node-backup", + "kind": "backup", + "content": "path ${dir:html}\n" + } + ] } diff --git a/modules/postgres/module.json b/modules/postgres/module.json index f25d84d..dc1a256 100644 --- a/modules/postgres/module.json +++ b/modules/postgres/module.json @@ -74,6 +74,13 @@ "mode": "0700", "owner": "999:70" }, + { + "id": "dumps", + "type": "directory", + "path": "${dir:store-data}/dumps", + "mode": "0700", + "owner": "999:70" + }, { "id": "server", "type": "container", @@ -121,5 +128,12 @@ } } ] - } + }, + "contributions": [ + { + "seat": "node-backup", + "kind": "backup", + "content": "run docker exec -u postgres postgres sh -c 'cd /var/lib/postgresql/data/dumps && for db in $(psql -Atc \"select datname from pg_database where oid >= 16384 order by 1\"); do pg_dump -Fc -f \"$db.dump.partial\" \"$db\" && mv \"$db.dump.partial\" \"$db.dump\" || exit 1; done'\npath ${dir:dumps}\n" + } + ] } diff --git a/modules/restic/client.ts b/modules/restic/client.ts new file mode 100644 index 0000000..884f2af --- /dev/null +++ b/modules/restic/client.ts @@ -0,0 +1,312 @@ +// restic's own code, in the module (novox/hq ADR 0039): the machine's backups (ADR 0214, to-be 43). +// +// The mesh composes what to back up: every module on the machine contributes `backup` lines to the +// node-backup seat, and the mesh writes them, each module's under a `# ` line and with its +// directories already filled, into one file this module reads. Two kinds of line: +// +// run run as root before the module's snapshot — a consistent dump of a store +// path a directory the module's snapshot keeps +// +// Each module gets one snapshot a night, tagged with its name, so a module is listed, kept and +// restored on its own. Everything lands in one repository on the machine — deduplicated, so every +// night is a complete restore point and only what changed costs space — and is thinned to 14 daily, +// 8 weekly and 6 monthly. Against mistakes, not disasters: nothing leaves the machine. +// +// Root's: the dumps read every store and the repository holds every module's data, and the runtime +// loading this bundle runs as the operator's account, so restic and the run lines go through sudo +// without a prompt where the account is not root — fail2ban's way (ADR 0175 §4). + +import { execFile } from "node:child_process"; +import { existsSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { promisify } from "node:util"; + +const execFileP = promisify(execFile); + +/** A command runner, so the backups can be tested without restic or a store. */ +export type Runner = (cmd: string, args: string[]) => Promise; + +/** The command as it is run: as given when this process is root, else through sudo without a prompt. */ +export function escalated(cmd: string, args: string[], uid: number | undefined = process.getuid?.()): [string, string[]] { + if (uid === 0) return [cmd, args]; + return ["sudo", ["-n", cmd, ...args]]; +} + +export const execRunner: Runner = async (cmd, args) => { + const [program, argv] = escalated(cmd, args); + try { + // A night's dump of a large store takes a while; six hours is a dump that will not finish. + const { stdout } = await execFileP(program, argv, { maxBuffer: 256 * 1024 * 1024, timeout: 6 * 3600 * 1000 }); + return stdout; + } catch (err) { + const e = err as { code?: string | number; stderr?: string; stdout?: string; message?: string }; + const said = `${e.stderr ?? ""}`.trim() || `${e.stdout ?? ""}`.trim(); + if (program === "sudo" && /^sudo:/m.test(said)) { + throw new Error(`${cmd} needs root and the runtime's account may not run it without a prompt: ${said}`); + } + const lines = said.split("\n").map((l) => l.trim()).filter(Boolean); + throw new Error(lines.length ? lines.slice(-3).join(" / ") : (e.message ?? `${cmd} failed`)); + } +}; + +/** What one module declared. */ +export interface Declared { + module: string; + runs: string[]; + paths: string[]; +} + +/** The composed file, read into each module's declaration, in the order the mesh wrote them. A line + * before any module, a comment that is not a module's name, or a blank, is nothing. */ +export function parseDeclared(text: string): Declared[] { + const out: Declared[] = []; + let current: Declared | undefined; + for (const raw of text.split("\n")) { + const line = raw.trim(); + if (line === "") continue; + const header = /^#\s*([a-z0-9][a-z0-9-]*)$/.exec(line); + if (header) { + current = { module: header[1], runs: [], paths: [] }; + out.push(current); + continue; + } + if (line.startsWith("#") || !current) continue; + const [kind, ...rest] = line.split(/\s+/); + const value = line.slice(kind.length).trim(); + if (kind === "run" && value) current.runs.push(value); + else if (kind === "path" && rest.length === 1 && value.startsWith("/")) current.paths.push(value); + else throw new Error(`${current.module} contributes a backup line this holder does not read: ${line}`); + } + return out.filter((d) => d.runs.length > 0 || d.paths.length > 0); +} + +/** One restore point, as restic lists it. */ +export interface Snapshot { + id: string; + short_id: string; + time: string; + paths: string[]; + tags?: string[]; + hostname: string; +} + +/** How one module's last night went. */ +export interface Night { + ok: boolean; + at: string; + snapshot?: string; + error?: string; +} + +export const KEEP = { daily: 14, weekly: 8, monthly: 6 }; + +export function tagOf(module: string): string { + return `module=${module}`; +} + +export interface Where { + declared: string; + repository: string; + passwordFile: string; + state: string; +} + +export function whereFromEnv(env: NodeJS.ProcessEnv = process.env): Where { + const need = (k: string) => { + const v = env[k]; + if (!v) throw new Error(`${k} is not set; the mesh gives it to this module's tools`); + return v; + }; + return { + declared: need("MESH_BACKUP_DECLARED"), + repository: need("MESH_BACKUP_REPOSITORY"), + passwordFile: need("MESH_BACKUP_PASSWORD_FILE"), + state: need("MESH_BACKUP_STATE"), + }; +} + +export class Backups { + private busy: Promise = Promise.resolve(); + readonly where: Where; + private run: Runner; + private readonly now: () => Date; + private readonly say: (line: string) => void; + + constructor( + where: Where, + run: Runner = execRunner, + now: () => Date = () => new Date(), + say: (line: string) => void = (l) => console.error(`[restic] ${l}`), + ) { + this.where = where; + this.run = run; + this.now = now; + this.say = say; + } + + private restic(args: string[]): Promise { + return this.run("restic", ["--repo", this.where.repository, "--password-file", this.where.passwordFile, "--no-cache", ...args]); + } + + /** One thing at a time: two nights, or a night and a restore, never share a dump. */ + private serial(work: () => Promise): Promise { + const next = this.busy.then(work, work); + this.busy = next.catch(() => undefined); + return next; + } + + declared(): Declared[] { + return parseDeclared(readFileSync(this.where.declared, "utf8")); + } + + private nightsFile(): string { + return join(this.where.state, "nights.json"); + } + + nights(): Record { + try { + return JSON.parse(readFileSync(this.nightsFile(), "utf8")) as Record; + } catch { + return {}; + } + } + + private record(module: string, night: Night): void { + const all = this.nights(); + all[module] = night; + writeFileSync(this.nightsFile(), JSON.stringify(all, null, 2) + "\n", { mode: 0o600 }); + } + + /** The repository, made the first time. A repository that exists and cannot be opened is said, + * never replaced: replacing it would discard every restore point to fix a password. */ + async ensureRepository(): Promise { + try { + await this.restic(["cat", "config"]); + } catch (err) { + const why = String((err as Error).message); + if (!/does not exist|unable to open config file|Is there a repository at the following location/i.test(why)) { + throw new Error(`the repository at ${this.where.repository} cannot be opened, and is left as it is: ${why}`); + } + this.say(`no repository at ${this.where.repository}; making one`); + await this.restic(["init"]); + } + } + + /** One module's night: its run lines, then one snapshot of its paths. A failure is that module's. */ + private async one(d: Declared): Promise { + const at = this.now().toISOString(); + try { + for (const command of d.runs) { + await this.run("sh", ["-c", command]); + } + const missing = d.paths.filter((p) => !existsSync(p)); + if (missing.length > 0) throw new Error(`${missing.join(", ")} does not exist`); + if (d.paths.length === 0) throw new Error("it runs a dump and names no directory to keep it from"); + const out = await this.restic(["backup", "--json", "--tag", tagOf(d.module), ...d.paths]); + const summary = out + .split("\n") + .map((l) => { try { return JSON.parse(l) as { message_type?: string; snapshot_id?: string }; } catch { return {}; } }) + .find((m) => m.message_type === "summary"); + const night: Night = { ok: true, at, snapshot: summary?.snapshot_id?.slice(0, 8) }; + this.say(`${d.module}: backed up (${night.snapshot ?? "no snapshot id reported"})`); + return night; + } catch (err) { + const night: Night = { ok: false, at, error: String((err as Error).message) }; + this.say(`${d.module}: NOT backed up: ${night.error}`); + return night; + } + } + + /** A night: every module, or one, then the rotation. Returns each module's outcome. */ + backUp(only?: string): Promise> { + return this.serial(async () => { + await this.ensureRepository(); + const all = this.declared(); + const chosen = only ? all.filter((d) => d.module === only) : all; + if (only && chosen.length === 0) { + throw new Error(`${only} declares nothing to back up on this machine; it backs up ${all.map((d) => d.module).join(", ") || "nothing"}`); + } + const outcome: Record = {}; + for (const d of chosen) { + outcome[d.module] = await this.one(d); + this.record(d.module, outcome[d.module]); + } + await this.restic([ + "forget", "--prune", "--group-by", "host,tags", + "--keep-daily", String(KEEP.daily), "--keep-weekly", String(KEEP.weekly), "--keep-monthly", String(KEEP.monthly), + ]).catch((err) => this.say(`thinning the restore points failed, and every one is kept: ${(err as Error).message}`)); + return outcome; + }); + } + + async snapshots(module?: string): Promise { + const args = ["snapshots", "--json"]; + if (module) args.push("--tag", tagOf(module)); + return JSON.parse((await this.restic(args)) || "[]") as Snapshot[]; + } + + /** What is backed up here: each module, what it declared, its last night and its restore points. */ + async backedUp(module?: string) { + const nights = this.nights(); + const snaps = await this.snapshots(module).catch(() => [] as Snapshot[]); + return this.declared() + .filter((d) => !module || d.module === module) + .map((d) => { + const mine = snaps.filter((s) => (s.tags ?? []).includes(tagOf(d.module))); + return { + module: d.module, + runs: d.runs.length, + paths: d.paths, + lastNight: nights[d.module] ?? null, + restorePoints: mine.length, + newest: mine.length ? { snapshot: mine[mine.length - 1].short_id, at: mine[mine.length - 1].time } : null, + }; + }); + } + + /** A module's data from a restore point, BESIDE the live data: each directory as + * .restored-. A target that already exists is refused, never overwritten. */ + restore(module: string, snapshot?: string, path?: string) { + return this.serial(async () => { + const mine = await this.snapshots(module); + if (mine.length === 0) throw new Error(`${module} has no restore point on this machine`); + const chosen = snapshot ? mine.find((s) => s.id.startsWith(snapshot) || s.short_id === snapshot) : mine[mine.length - 1]; + if (!chosen) { + throw new Error(`${module} has no restore point ${snapshot}; it has ${mine.map((s) => `${s.short_id} (${s.time})`).join(", ")}`); + } + const paths = path ? chosen.paths.filter((p) => p === path) : chosen.paths; + if (paths.length === 0) throw new Error(`restore point ${chosen.short_id} of ${module} holds ${chosen.paths.join(", ")}, not ${path}`); + const stamp = this.now().toISOString().replace(/[-:]/g, "").replace("T", "-").slice(0, 15); + const restored: string[] = []; + for (const p of paths) { + const target = `${p}.restored-${stamp}`; + if (existsSync(target)) throw new Error(`${target} already exists; nothing is restored over anything`); + await this.restic(["restore", `${chosen.id}:${p}`, "--target", target]); + restored.push(target); + this.say(`${module}: restored ${p} from ${chosen.short_id} to ${target}`); + } + return { module, from: { snapshot: chosen.short_id, at: chosen.time }, restored, live: "untouched — swapping it in is a person's act" }; + }); + } + + /** The weekly look at the repository's own integrity, with a sample of the data read back. */ + check(): Promise { + return this.serial(() => this.restic(["check", "--read-data-subset", "5%"])); + } +} + +/** When the next night is due: the given hour, local time, today if it is still ahead, else tomorrow. */ +export function nextNight(now: Date, hour: number): Date { + const next = new Date(now); + next.setHours(hour, 0, 0, 0); + if (next.getTime() <= now.getTime()) next.setDate(next.getDate() + 1); + return next; +} + +/** Whether a night was missed: the newest good night of any module is older than a day and a bit — + * the machine was off, or this module was not running, at the hour. */ +export function missedANight(nights: Record, now: Date): boolean { + const good = Object.values(nights).filter((n) => n.ok).map((n) => Date.parse(n.at)); + if (good.length === 0) return true; + return now.getTime() - Math.max(...good) > 26 * 3600 * 1000; +} diff --git a/modules/restic/module.json b/modules/restic/module.json new file mode 100644 index 0000000..f76a457 --- /dev/null +++ b/modules/restic/module.json @@ -0,0 +1,64 @@ +{ + "module": "restic", + "version": "1", + "claims": [ + { + "name": "node-backup", + "scope": "node", + "serves": [ + "backed-up", + "now", + "restore" + ] + } + ], + "own-secrets": { + "repository": "${dir:state}/repository.secret" + }, + "resources": [ + { + "id": "state", + "type": "directory", + "mode": "0700", + "place": "." + }, + { + "id": "repository", + "type": "directory", + "mode": "0700" + }, + { + "id": "declared", + "type": "file", + "path": "${dir:state}/backups.conf", + "mode": "0600", + "content": "# What the modules on this machine back up, composed by the mesh (novox/hq to-be 43). Do not edit.\n${contribution:node-backup:backup}" + }, + { + "id": "tool", + "type": "package", + "package": "restic" + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "typescript", + "entrypoints": [ + "tools/index.js" + ], + "loads": [ + "tools/index.js" + ], + "env": { + "MESH_BACKUP_DECLARED": "${dir:state}/backups.conf", + "MESH_BACKUP_REPOSITORY": "${dir:repository}", + "MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret", + "MESH_BACKUP_STATE": "${dir:state}" + } + } + ] + } +} diff --git a/modules/restic/package.json b/modules/restic/package.json new file mode 100644 index 0000000..bdcffc0 --- /dev/null +++ b/modules/restic/package.json @@ -0,0 +1,18 @@ +{ + "name": "@novox/module-restic", + "version": "0.1.0", + "description": "restic \u2014 the machine's backups: holds the node-backup seat, takes a nightly restore point of what every module on the machine declares, keeps 14 daily, 8 weekly and 6 monthly, and restores beside the live data (novox/hq ADR 0214, to-be 43).", + "type": "module", + "private": true, + "dependencies": { + "@novox/mesh-sdk": "^0.1.1" + }, + "devDependencies": { + "@types/node": "^22.0.0", + "typescript": "^5.6.0" + }, + "scripts": { + "build": "tsc client.ts tools/index.ts --module NodeNext --moduleResolution NodeNext --target ES2022 --rootDir . --outDir dist", + "test": "node --test --experimental-strip-types 'test/*.test.ts'" + } +} diff --git a/modules/restic/test/client.test.ts b/modules/restic/test/client.test.ts new file mode 100644 index 0000000..599dd2f --- /dev/null +++ b/modules/restic/test/client.test.ts @@ -0,0 +1,142 @@ +// The machine's backups over a fake restic and fake stores (novox/hq ADR 0214, to-be 43), and — where +// restic is installed — over the real one, on throwaway directories. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Backups, escalated, missedANight, nextNight, parseDeclared, type Runner } from "../client.ts"; + +const COMPOSED = + "# What the modules on this machine back up, composed by the mesh. Do not edit.\n" + + "# postgres\nrun docker exec -u postgres postgres sh -c 'pg-dump-all'\npath /var/lib/mesh-store/dumps\n" + + "# mailu\npath /var/lib/mailu/data-mail\npath /var/lib/mailu/data-dkim\n"; + +function place(declared: string) { + const dir = mkdtempSync(join(tmpdir(), "restic-test-")); + writeFileSync(join(dir, "backups.conf"), declared); + writeFileSync(join(dir, "pw"), "secret\n"); + return { declared: join(dir, "backups.conf"), repository: join(dir, "repo"), passwordFile: join(dir, "pw"), state: dir }; +} + +test("the composed file is read into each module's runs and paths, the header comment being nothing", () => { + assert.deepEqual(parseDeclared(COMPOSED), [ + { module: "postgres", runs: ["docker exec -u postgres postgres sh -c 'pg-dump-all'"], paths: ["/var/lib/mesh-store/dumps"] }, + { module: "mailu", runs: [], paths: ["/var/lib/mailu/data-mail", "/var/lib/mailu/data-dkim"] }, + ]); +}); + +test("a line this holder does not read is refused, naming the module, rather than skipped", () => { + assert.throws(() => parseDeclared("# pg\ncopy /x\n"), /pg contributes a backup line this holder does not read: copy \/x/); + assert.throws(() => parseDeclared("# pg\npath relative/dir\n"), /pg contributes/); +}); + +test("restic and the dumps run through sudo where the account is not root", () => { + assert.deepEqual(escalated("restic", ["snapshots"], 1000), ["sudo", ["-n", "restic", "snapshots"]]); + assert.deepEqual(escalated("restic", ["snapshots"], 0), ["restic", ["snapshots"]]); +}); + +test("a night runs each module's dump before its snapshot, and one failing module fails only itself", async () => { + const where = place("# pg\nrun dump-it\npath /\n# broken\nrun fail-it\npath /\n# mail\npath /\n"); + const calls: string[] = []; + const run: Runner = async (cmd, args) => { + const line = cmd === "restic" ? `restic ${args.slice(5).join(" ")}` : `${cmd} ${args.join(" ")}`; + calls.push(line); + if (line === "sh -c fail-it") throw new Error("the dump failed"); + if (line.startsWith("restic backup")) return '{"message_type":"status"}\n{"message_type":"summary","snapshot_id":"abcdef0123456789"}\n'; + return ""; + }; + const outcome = await new Backups(where, run, () => new Date("2026-10-06T03:00:00Z"), () => {}).backUp(); + assert.equal(outcome.pg.ok, true); + assert.equal(outcome.pg.snapshot, "abcdef01"); + assert.equal(outcome.broken.ok, false); + assert.match(outcome.broken.error ?? "", /the dump failed/); + assert.equal(outcome.mail.ok, true); + assert.deepEqual(calls, [ + "restic cat config", + "sh -c dump-it", + "restic backup --json --tag module=pg /", + "sh -c fail-it", + "restic backup --json --tag module=mail /", + "restic forget --prune --group-by host,tags --keep-daily 14 --keep-weekly 8 --keep-monthly 6", + ]); + // Recorded, so `backed-up` and the missed-night check read it. + const nights = JSON.parse(readFileSync(join(where.state, "nights.json"), "utf8")); + assert.equal(nights.broken.ok, false); + assert.equal(nights.mail.ok, true); +}); + +test("a repository that exists and will not open is never replaced", async () => { + const where = place("# pg\npath /\n"); + const calls: string[] = []; + const run: Runner = async (cmd, args) => { + calls.push(args.slice(5).join(" ")); + if (args.includes("cat")) throw new Error("Fatal: wrong password or no key found"); + return ""; + }; + await assert.rejects(new Backups(where, run, undefined, () => {}).backUp(), /cannot be opened, and is left as it is/); + assert.ok(!calls.includes("init"), "it made a new repository over one it could not open"); +}); + +test("a declared directory that does not exist fails that module's night", async () => { + const where = place("# pg\npath /nowhere/at/all\n"); + const run: Runner = async () => ""; + const outcome = await new Backups(where, run, undefined, () => {}).backUp(); + assert.equal(outcome.pg.ok, false); + assert.match(outcome.pg.error ?? "", /\/nowhere\/at\/all does not exist/); +}); + +test("a night is due at the hour today while it is ahead, else tomorrow; a missed one is noticed", () => { + const morning = new Date(2026, 9, 6, 1, 30); + assert.equal(nextNight(morning, 3).getTime(), new Date(2026, 9, 6, 3, 0).getTime()); + const afternoon = new Date(2026, 9, 6, 15, 0); + assert.equal(nextNight(afternoon, 3).getTime(), new Date(2026, 9, 7, 3, 0).getTime()); + assert.equal(missedANight({}, afternoon), true); + assert.equal(missedANight({ pg: { ok: true, at: new Date(2026, 9, 6, 3, 5).toISOString() } }, afternoon), false); + assert.equal(missedANight({ pg: { ok: true, at: new Date(2026, 9, 4, 3, 5).toISOString() } }, afternoon), true); + assert.equal(missedANight({ pg: { ok: false, at: new Date(2026, 9, 6, 3, 5).toISOString() } }, afternoon), true); +}); + +// The real thing, where restic is installed: a dump, a snapshot, a mistake, and a restore beside. +const hasRestic = (() => { + try { + execFileSync("restic", ["version"], { stdio: "ignore" }); + return true; + } catch { + return false; + } +})(); + +test("with the real restic: a mistake is undone by a restore beside the live data", { skip: !hasRestic && "restic is not installed" }, async () => { + const root = mkdtempSync(join(tmpdir(), "restic-real-")); + const store = join(root, "store"); + const dumps = join(root, "dumps"); + mkdirSync(store); + mkdirSync(dumps); + writeFileSync(join(store, "mailbox"), "the only copy of a letter\n"); + const where = place(`# mail\npath ${store}\n# pg\nrun echo 'every row' > ${dumps}/all.dump\npath ${dumps}\n`); + const backups = new Backups(where, undefined, () => new Date("2026-10-06T03:00:00Z"), () => {}); + // escalated() would sudo; the test runs as whoever it is, against its own repository. + (backups as unknown as { run: Runner }).run = async (cmd, args) => execFileSync(cmd, args, { encoding: "utf8" }); + + const night = await backups.backUp(); + assert.equal(night.mail.ok, true, night.mail.error); + assert.equal(night.pg.ok, true, night.pg.error); + assert.equal(readFileSync(join(dumps, "all.dump"), "utf8"), "every row\n"); + + // The mistake. + rmSync(join(store, "mailbox")); + + const listed = await backups.backedUp(); + assert.deepEqual(listed.map((m) => [m.module, m.restorePoints]), [["mail", 1], ["pg", 1]]); + + const restored = await backups.restore("mail"); + assert.equal(restored.restored.length, 1); + assert.match(restored.restored[0], /store\.restored-20261006-030000$/); + assert.equal(readFileSync(join(restored.restored[0], "mailbox"), "utf8"), "the only copy of a letter\n"); + assert.ok(!existsSync(join(store, "mailbox")), "the restore wrote into the live directory"); + + // Never over anything: the same restore again finds its target taken. + await assert.rejects(backups.restore("mail"), /already exists; nothing is restored over anything/); +}); diff --git a/modules/restic/tools/index.ts b/modules/restic/tools/index.ts new file mode 100644 index 0000000..95f15c1 --- /dev/null +++ b/modules/restic/tools/index.ts @@ -0,0 +1,78 @@ +// The machine's backups: the node-backup seat's three verbs — what is backed up, take one now, +// restore beside the live data — and the night that runs without anyone asking (novox/hq ADR 0214, +// to-be 43). What is backed up is composed by the mesh from the modules the machine runs; this code +// only runs it. + +import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools"; +import { Backups, missedANight, nextNight, whereFromEnv } from "../client.js"; + +export function getSeatVerbs(backups: Backups): ToolDefinition[] { + return [ + { + name: "backed-up", + description: + "What this machine backs up: each module, what it declared, its last good night, how many restore points are kept.", + input: { module: { type: "string", description: "one module (optional)" } }, + run: async (args) => backups.backedUp(args.module ? String(args.module) : undefined), + }, + { + name: "now", + description: + "Take a backup now, of one module or of every module on this machine — before a migration, a retirement or anything else that could go wrong. Answers when it has started; `backed-up` says how it went.", + input: { module: { type: "string", description: "one module (optional)" } }, + run: async (args) => { + const only = args.module ? String(args.module) : undefined; + // A night of a large store outlasts any call; it is started, and its outcome recorded. + backups.backUp(only).catch((err) => console.error(`[restic] a backup asked for now failed: ${(err as Error).message}`)); + return { started: only ?? "every module on this machine", follow: "backed-up" }; + }, + }, + { + name: "restore", + description: + "Restore one module's data from a restore point BESIDE the live data, never over it: each directory as .restored-. Swapping it in is a person's act.", + input: { + module: { type: "string", description: "the module" }, + snapshot: { type: "string", description: "the restore point (the newest when omitted)" }, + path: { type: "string", description: "one of the module's directories (all of them when omitted)" }, + }, + run: async (args) => + backups.restore(String(args.module ?? ""), args.snapshot ? String(args.snapshot) : undefined, args.path ? String(args.path) : undefined), + }, + ]; +} + +const backups = new Backups(whereFromEnv()); +registerModuleTools("node-backup", () => getSeatVerbs(backups)); + +// The night. At the hour, every module; and at start, if a night was missed — the machine was off +// or this module was not running at the hour — one now rather than a day later. +const HOUR = Number(process.env.MESH_BACKUP_HOUR ?? "3"); + +async function night(): Promise { + try { + const outcome = await backups.backUp(); + const failed = Object.entries(outcome).filter(([, n]) => !n.ok).map(([m]) => m); + if (failed.length > 0) console.error(`[restic] the night left ${failed.join(", ")} without a backup`); + // Sundays, the repository's own integrity with a sample of the data read back. + if (new Date().getDay() === 0) { + await backups.check().then( + () => console.error("[restic] the repository checks out"), + (err) => console.error(`[restic] the repository does NOT check out: ${(err as Error).message}`), + ); + } + } catch (err) { + console.error(`[restic] the night did not run: ${(err as Error).message}`); + } +} + +function schedule(): void { + const at = nextNight(new Date(), HOUR); + setTimeout(() => void night().finally(schedule), at.getTime() - Date.now()); +} + +if (missedANight(backups.nights(), new Date())) { + // Not at once: a machine just started has its stores still coming up. + setTimeout(() => void night(), 10 * 60 * 1000); +} +schedule(); diff --git a/modules/restic/tsconfig.json b/modules/restic/tsconfig.json new file mode 100644 index 0000000..1f1b70a --- /dev/null +++ b/modules/restic/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "noEmit": true + }, + "include": [ + "client.ts", + "tools/index.ts" + ] +} From 3a88bca21ae31a9c1ed29763f447196d247f393a Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 5 Oct 2026 11:58:18 +0200 Subject: [PATCH 2/2] restic: the holder in Go Go is the default for new module code; the holder is one binary like the licence manager, serving the seat's verbs over the SDK and running the nights beside them. Same behaviour and tests. --- modules/restic/client.ts | 312 ------------ modules/restic/cmd/restic-backups/backups.go | 472 ++++++++++++++++++ .../restic/cmd/restic-backups/backups_test.go | 228 +++++++++ modules/restic/cmd/restic-backups/main.go | 137 +++++ modules/restic/go.mod | 5 + modules/restic/go.sum | 2 + modules/restic/module.json | 10 +- modules/restic/package.json | 18 - modules/restic/test/client.test.ts | 142 ------ modules/restic/tools/index.ts | 78 --- modules/restic/tsconfig.json | 15 - 11 files changed, 849 insertions(+), 570 deletions(-) delete mode 100644 modules/restic/client.ts create mode 100644 modules/restic/cmd/restic-backups/backups.go create mode 100644 modules/restic/cmd/restic-backups/backups_test.go create mode 100644 modules/restic/cmd/restic-backups/main.go create mode 100644 modules/restic/go.mod create mode 100644 modules/restic/go.sum delete mode 100644 modules/restic/package.json delete mode 100644 modules/restic/test/client.test.ts delete mode 100644 modules/restic/tools/index.ts delete mode 100644 modules/restic/tsconfig.json diff --git a/modules/restic/client.ts b/modules/restic/client.ts deleted file mode 100644 index 884f2af..0000000 --- a/modules/restic/client.ts +++ /dev/null @@ -1,312 +0,0 @@ -// restic's own code, in the module (novox/hq ADR 0039): the machine's backups (ADR 0214, to-be 43). -// -// The mesh composes what to back up: every module on the machine contributes `backup` lines to the -// node-backup seat, and the mesh writes them, each module's under a `# ` line and with its -// directories already filled, into one file this module reads. Two kinds of line: -// -// run run as root before the module's snapshot — a consistent dump of a store -// path a directory the module's snapshot keeps -// -// Each module gets one snapshot a night, tagged with its name, so a module is listed, kept and -// restored on its own. Everything lands in one repository on the machine — deduplicated, so every -// night is a complete restore point and only what changed costs space — and is thinned to 14 daily, -// 8 weekly and 6 monthly. Against mistakes, not disasters: nothing leaves the machine. -// -// Root's: the dumps read every store and the repository holds every module's data, and the runtime -// loading this bundle runs as the operator's account, so restic and the run lines go through sudo -// without a prompt where the account is not root — fail2ban's way (ADR 0175 §4). - -import { execFile } from "node:child_process"; -import { existsSync, readFileSync, writeFileSync } from "node:fs"; -import { join } from "node:path"; -import { promisify } from "node:util"; - -const execFileP = promisify(execFile); - -/** A command runner, so the backups can be tested without restic or a store. */ -export type Runner = (cmd: string, args: string[]) => Promise; - -/** The command as it is run: as given when this process is root, else through sudo without a prompt. */ -export function escalated(cmd: string, args: string[], uid: number | undefined = process.getuid?.()): [string, string[]] { - if (uid === 0) return [cmd, args]; - return ["sudo", ["-n", cmd, ...args]]; -} - -export const execRunner: Runner = async (cmd, args) => { - const [program, argv] = escalated(cmd, args); - try { - // A night's dump of a large store takes a while; six hours is a dump that will not finish. - const { stdout } = await execFileP(program, argv, { maxBuffer: 256 * 1024 * 1024, timeout: 6 * 3600 * 1000 }); - return stdout; - } catch (err) { - const e = err as { code?: string | number; stderr?: string; stdout?: string; message?: string }; - const said = `${e.stderr ?? ""}`.trim() || `${e.stdout ?? ""}`.trim(); - if (program === "sudo" && /^sudo:/m.test(said)) { - throw new Error(`${cmd} needs root and the runtime's account may not run it without a prompt: ${said}`); - } - const lines = said.split("\n").map((l) => l.trim()).filter(Boolean); - throw new Error(lines.length ? lines.slice(-3).join(" / ") : (e.message ?? `${cmd} failed`)); - } -}; - -/** What one module declared. */ -export interface Declared { - module: string; - runs: string[]; - paths: string[]; -} - -/** The composed file, read into each module's declaration, in the order the mesh wrote them. A line - * before any module, a comment that is not a module's name, or a blank, is nothing. */ -export function parseDeclared(text: string): Declared[] { - const out: Declared[] = []; - let current: Declared | undefined; - for (const raw of text.split("\n")) { - const line = raw.trim(); - if (line === "") continue; - const header = /^#\s*([a-z0-9][a-z0-9-]*)$/.exec(line); - if (header) { - current = { module: header[1], runs: [], paths: [] }; - out.push(current); - continue; - } - if (line.startsWith("#") || !current) continue; - const [kind, ...rest] = line.split(/\s+/); - const value = line.slice(kind.length).trim(); - if (kind === "run" && value) current.runs.push(value); - else if (kind === "path" && rest.length === 1 && value.startsWith("/")) current.paths.push(value); - else throw new Error(`${current.module} contributes a backup line this holder does not read: ${line}`); - } - return out.filter((d) => d.runs.length > 0 || d.paths.length > 0); -} - -/** One restore point, as restic lists it. */ -export interface Snapshot { - id: string; - short_id: string; - time: string; - paths: string[]; - tags?: string[]; - hostname: string; -} - -/** How one module's last night went. */ -export interface Night { - ok: boolean; - at: string; - snapshot?: string; - error?: string; -} - -export const KEEP = { daily: 14, weekly: 8, monthly: 6 }; - -export function tagOf(module: string): string { - return `module=${module}`; -} - -export interface Where { - declared: string; - repository: string; - passwordFile: string; - state: string; -} - -export function whereFromEnv(env: NodeJS.ProcessEnv = process.env): Where { - const need = (k: string) => { - const v = env[k]; - if (!v) throw new Error(`${k} is not set; the mesh gives it to this module's tools`); - return v; - }; - return { - declared: need("MESH_BACKUP_DECLARED"), - repository: need("MESH_BACKUP_REPOSITORY"), - passwordFile: need("MESH_BACKUP_PASSWORD_FILE"), - state: need("MESH_BACKUP_STATE"), - }; -} - -export class Backups { - private busy: Promise = Promise.resolve(); - readonly where: Where; - private run: Runner; - private readonly now: () => Date; - private readonly say: (line: string) => void; - - constructor( - where: Where, - run: Runner = execRunner, - now: () => Date = () => new Date(), - say: (line: string) => void = (l) => console.error(`[restic] ${l}`), - ) { - this.where = where; - this.run = run; - this.now = now; - this.say = say; - } - - private restic(args: string[]): Promise { - return this.run("restic", ["--repo", this.where.repository, "--password-file", this.where.passwordFile, "--no-cache", ...args]); - } - - /** One thing at a time: two nights, or a night and a restore, never share a dump. */ - private serial(work: () => Promise): Promise { - const next = this.busy.then(work, work); - this.busy = next.catch(() => undefined); - return next; - } - - declared(): Declared[] { - return parseDeclared(readFileSync(this.where.declared, "utf8")); - } - - private nightsFile(): string { - return join(this.where.state, "nights.json"); - } - - nights(): Record { - try { - return JSON.parse(readFileSync(this.nightsFile(), "utf8")) as Record; - } catch { - return {}; - } - } - - private record(module: string, night: Night): void { - const all = this.nights(); - all[module] = night; - writeFileSync(this.nightsFile(), JSON.stringify(all, null, 2) + "\n", { mode: 0o600 }); - } - - /** The repository, made the first time. A repository that exists and cannot be opened is said, - * never replaced: replacing it would discard every restore point to fix a password. */ - async ensureRepository(): Promise { - try { - await this.restic(["cat", "config"]); - } catch (err) { - const why = String((err as Error).message); - if (!/does not exist|unable to open config file|Is there a repository at the following location/i.test(why)) { - throw new Error(`the repository at ${this.where.repository} cannot be opened, and is left as it is: ${why}`); - } - this.say(`no repository at ${this.where.repository}; making one`); - await this.restic(["init"]); - } - } - - /** One module's night: its run lines, then one snapshot of its paths. A failure is that module's. */ - private async one(d: Declared): Promise { - const at = this.now().toISOString(); - try { - for (const command of d.runs) { - await this.run("sh", ["-c", command]); - } - const missing = d.paths.filter((p) => !existsSync(p)); - if (missing.length > 0) throw new Error(`${missing.join(", ")} does not exist`); - if (d.paths.length === 0) throw new Error("it runs a dump and names no directory to keep it from"); - const out = await this.restic(["backup", "--json", "--tag", tagOf(d.module), ...d.paths]); - const summary = out - .split("\n") - .map((l) => { try { return JSON.parse(l) as { message_type?: string; snapshot_id?: string }; } catch { return {}; } }) - .find((m) => m.message_type === "summary"); - const night: Night = { ok: true, at, snapshot: summary?.snapshot_id?.slice(0, 8) }; - this.say(`${d.module}: backed up (${night.snapshot ?? "no snapshot id reported"})`); - return night; - } catch (err) { - const night: Night = { ok: false, at, error: String((err as Error).message) }; - this.say(`${d.module}: NOT backed up: ${night.error}`); - return night; - } - } - - /** A night: every module, or one, then the rotation. Returns each module's outcome. */ - backUp(only?: string): Promise> { - return this.serial(async () => { - await this.ensureRepository(); - const all = this.declared(); - const chosen = only ? all.filter((d) => d.module === only) : all; - if (only && chosen.length === 0) { - throw new Error(`${only} declares nothing to back up on this machine; it backs up ${all.map((d) => d.module).join(", ") || "nothing"}`); - } - const outcome: Record = {}; - for (const d of chosen) { - outcome[d.module] = await this.one(d); - this.record(d.module, outcome[d.module]); - } - await this.restic([ - "forget", "--prune", "--group-by", "host,tags", - "--keep-daily", String(KEEP.daily), "--keep-weekly", String(KEEP.weekly), "--keep-monthly", String(KEEP.monthly), - ]).catch((err) => this.say(`thinning the restore points failed, and every one is kept: ${(err as Error).message}`)); - return outcome; - }); - } - - async snapshots(module?: string): Promise { - const args = ["snapshots", "--json"]; - if (module) args.push("--tag", tagOf(module)); - return JSON.parse((await this.restic(args)) || "[]") as Snapshot[]; - } - - /** What is backed up here: each module, what it declared, its last night and its restore points. */ - async backedUp(module?: string) { - const nights = this.nights(); - const snaps = await this.snapshots(module).catch(() => [] as Snapshot[]); - return this.declared() - .filter((d) => !module || d.module === module) - .map((d) => { - const mine = snaps.filter((s) => (s.tags ?? []).includes(tagOf(d.module))); - return { - module: d.module, - runs: d.runs.length, - paths: d.paths, - lastNight: nights[d.module] ?? null, - restorePoints: mine.length, - newest: mine.length ? { snapshot: mine[mine.length - 1].short_id, at: mine[mine.length - 1].time } : null, - }; - }); - } - - /** A module's data from a restore point, BESIDE the live data: each directory as - * .restored-. A target that already exists is refused, never overwritten. */ - restore(module: string, snapshot?: string, path?: string) { - return this.serial(async () => { - const mine = await this.snapshots(module); - if (mine.length === 0) throw new Error(`${module} has no restore point on this machine`); - const chosen = snapshot ? mine.find((s) => s.id.startsWith(snapshot) || s.short_id === snapshot) : mine[mine.length - 1]; - if (!chosen) { - throw new Error(`${module} has no restore point ${snapshot}; it has ${mine.map((s) => `${s.short_id} (${s.time})`).join(", ")}`); - } - const paths = path ? chosen.paths.filter((p) => p === path) : chosen.paths; - if (paths.length === 0) throw new Error(`restore point ${chosen.short_id} of ${module} holds ${chosen.paths.join(", ")}, not ${path}`); - const stamp = this.now().toISOString().replace(/[-:]/g, "").replace("T", "-").slice(0, 15); - const restored: string[] = []; - for (const p of paths) { - const target = `${p}.restored-${stamp}`; - if (existsSync(target)) throw new Error(`${target} already exists; nothing is restored over anything`); - await this.restic(["restore", `${chosen.id}:${p}`, "--target", target]); - restored.push(target); - this.say(`${module}: restored ${p} from ${chosen.short_id} to ${target}`); - } - return { module, from: { snapshot: chosen.short_id, at: chosen.time }, restored, live: "untouched — swapping it in is a person's act" }; - }); - } - - /** The weekly look at the repository's own integrity, with a sample of the data read back. */ - check(): Promise { - return this.serial(() => this.restic(["check", "--read-data-subset", "5%"])); - } -} - -/** When the next night is due: the given hour, local time, today if it is still ahead, else tomorrow. */ -export function nextNight(now: Date, hour: number): Date { - const next = new Date(now); - next.setHours(hour, 0, 0, 0); - if (next.getTime() <= now.getTime()) next.setDate(next.getDate() + 1); - return next; -} - -/** Whether a night was missed: the newest good night of any module is older than a day and a bit — - * the machine was off, or this module was not running, at the hour. */ -export function missedANight(nights: Record, now: Date): boolean { - const good = Object.values(nights).filter((n) => n.ok).map((n) => Date.parse(n.at)); - if (good.length === 0) return true; - return now.getTime() - Math.max(...good) > 26 * 3600 * 1000; -} diff --git a/modules/restic/cmd/restic-backups/backups.go b/modules/restic/cmd/restic-backups/backups.go new file mode 100644 index 0000000..a65869e --- /dev/null +++ b/modules/restic/cmd/restic-backups/backups.go @@ -0,0 +1,472 @@ +// The machine's backups (novox/hq ADR 0214, to-be 43). +// +// The mesh composes what to back up: every module on the machine contributes `backup` lines to the +// node-backup seat, and the mesh writes them, each module's under a `# ` line and with its +// directories already filled, into one file this module reads. Two kinds of line: +// +// run run as root before the module's snapshot — a consistent dump of a store +// path a directory the module's snapshot keeps +// +// Each module gets one snapshot a night, tagged with its name, so a module is listed, kept and +// restored on its own. Everything lands in one repository on the machine — deduplicated, so every +// night is a complete restore point and only what changed costs space — and is thinned to 14 daily, +// 8 weekly and 6 monthly. Against mistakes, not disasters: nothing leaves the machine. +// +// Root's: the dumps read every store and the repository holds every module's data, and the runtime +// launching this binary runs as the operator's account, so restic and the run lines go through sudo +// without a prompt where the account is not root (ADR 0175 §4). +package main + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "slices" + "strings" + "sync" + "time" +) + +// Runner runs one command and answers what it printed, so the backups can be tested without restic +// or a store. +type Runner func(ctx context.Context, name string, args ...string) (string, error) + +// escalated is the command as it is run: as given when this process is root, else through sudo +// without a prompt. +func escalated(uid int, name string, args []string) (string, []string) { + if uid == 0 { + return name, args + } + return "sudo", append([]string{"-n", name}, args...) +} + +// execRunner runs it for real. A night's dump of a large store takes a while; six hours is a dump +// that will not finish. +func execRunner(ctx context.Context, name string, args ...string) (string, error) { + ctx, cancel := context.WithTimeout(ctx, 6*time.Hour) + defer cancel() + program, argv := escalated(os.Getuid(), name, args) + var stdout, stderr bytes.Buffer + cmd := exec.CommandContext(ctx, program, argv...) + cmd.Stdout, cmd.Stderr = &stdout, &stderr + if err := cmd.Run(); err != nil { + said := strings.TrimSpace(stderr.String()) + if said == "" { + said = strings.TrimSpace(stdout.String()) + } + if program == "sudo" && strings.HasPrefix(said, "sudo:") { + return "", fmt.Errorf("%s needs root and the runtime's account may not run it without a prompt: %s", name, said) + } + lines := strings.Split(said, "\n") + if len(lines) > 3 { + lines = lines[len(lines)-3:] + } + if said == "" { + return "", fmt.Errorf("%s: %w", name, err) + } + return "", errors.New(strings.Join(lines, " / ")) + } + return stdout.String(), nil +} + +// Declared is what one module declared. +type Declared struct { + Module string `json:"module"` + Runs []string `json:"runs"` + Paths []string `json:"paths"` +} + +var moduleHeader = regexp.MustCompile(`^#\s*([a-z0-9][a-z0-9-]*)$`) + +// parseDeclared reads the composed file into each module's declaration, in the order the mesh wrote +// them. A line before any module, a comment that is not a module's name, or a blank, is nothing; a +// line this holder does not read is refused, naming the module, rather than skipped. +func parseDeclared(text string) ([]Declared, error) { + var out []Declared + current := -1 + for _, raw := range strings.Split(text, "\n") { + line := strings.TrimSpace(raw) + if line == "" { + continue + } + if m := moduleHeader.FindStringSubmatch(line); m != nil { + out = append(out, Declared{Module: m[1]}) + current = len(out) - 1 + continue + } + if strings.HasPrefix(line, "#") || current < 0 { + continue + } + kind, value, _ := strings.Cut(line, " ") + value = strings.TrimSpace(value) + d := &out[current] + switch { + case kind == "run" && value != "": + d.Runs = append(d.Runs, value) + case kind == "path" && strings.HasPrefix(value, "/") && !strings.ContainsAny(value, " \t"): + d.Paths = append(d.Paths, value) + default: + return nil, fmt.Errorf("%s contributes a backup line this holder does not read: %s", d.Module, line) + } + } + kept := out[:0] + for _, d := range out { + if len(d.Runs) > 0 || len(d.Paths) > 0 { + kept = append(kept, d) + } + } + return kept, nil +} + +// Snapshot is one restore point, as restic lists it. +type Snapshot struct { + ID string `json:"id"` + ShortID string `json:"short_id"` + Time time.Time `json:"time"` + Paths []string `json:"paths"` + Tags []string `json:"tags"` + Hostname string `json:"hostname"` +} + +// Night is how one module's last night went. +type Night struct { + OK bool `json:"ok"` + At time.Time `json:"at"` + Snapshot string `json:"snapshot,omitempty"` + Error string `json:"error,omitempty"` +} + +// Keep is the rotation (novox/hq ADR 0214). +var Keep = struct{ Daily, Weekly, Monthly int }{14, 8, 6} + +func tagOf(module string) string { return "module=" + module } + +// Where is where the mesh put this module's things. +type Where struct { + Declared, Repository, PasswordFile, State string +} + +func whereFromEnv() (Where, error) { + var missing []string + get := func(k string) string { + v := os.Getenv(k) + if v == "" { + missing = append(missing, k) + } + return v + } + w := Where{ + Declared: get("MESH_BACKUP_DECLARED"), + Repository: get("MESH_BACKUP_REPOSITORY"), + PasswordFile: get("MESH_BACKUP_PASSWORD_FILE"), + State: get("MESH_BACKUP_STATE"), + } + if len(missing) > 0 { + return w, fmt.Errorf("%s not set; the mesh gives them to this module", strings.Join(missing, ", ")) + } + return w, nil +} + +// Backups is the machine's backups. One thing at a time: two nights, or a night and a restore, never +// share a dump. +type Backups struct { + Where Where + Run Runner + Now func() time.Time + Say func(format string, args ...any) + mu sync.Mutex +} + +func (b *Backups) restic(ctx context.Context, args ...string) (string, error) { + return b.Run(ctx, "restic", append([]string{"--repo", b.Where.Repository, "--password-file", b.Where.PasswordFile, "--no-cache"}, args...)...) +} + +// Declared is what the modules on this machine declared. +func (b *Backups) Declared() ([]Declared, error) { + raw, err := os.ReadFile(b.Where.Declared) + if err != nil { + return nil, err + } + return parseDeclared(string(raw)) +} + +func (b *Backups) nightsFile() string { return filepath.Join(b.Where.State, "nights.json") } + +// Nights is how each module's last night went. +func (b *Backups) Nights() map[string]Night { + nights := map[string]Night{} + if raw, err := os.ReadFile(b.nightsFile()); err == nil { + _ = json.Unmarshal(raw, &nights) + } + return nights +} + +func (b *Backups) record(module string, n Night) { + nights := b.Nights() + nights[module] = n + raw, _ := json.MarshalIndent(nights, "", " ") + if err := os.WriteFile(b.nightsFile(), append(raw, '\n'), 0o600); err != nil { + b.Say("recording %s's night failed: %v", module, err) + } +} + +var noRepository = regexp.MustCompile(`(?i)does not exist|unable to open config file|Is there a repository at the following location`) + +// ensureRepository makes the repository the first time. One that exists and cannot be opened is +// said, never replaced: replacing it would discard every restore point to fix a password. +func (b *Backups) ensureRepository(ctx context.Context) error { + _, err := b.restic(ctx, "cat", "config") + if err == nil { + return nil + } + if !noRepository.MatchString(err.Error()) { + return fmt.Errorf("the repository at %s cannot be opened, and is left as it is: %v", b.Where.Repository, err) + } + b.Say("no repository at %s; making one", b.Where.Repository) + _, err = b.restic(ctx, "init") + return err +} + +// one is one module's night: its run lines, then one snapshot of its paths. A failure is that +// module's alone. +func (b *Backups) one(ctx context.Context, d Declared) Night { + n := Night{At: b.Now().UTC()} + fail := func(err error) Night { + n.Error = err.Error() + b.Say("%s: NOT backed up: %s", d.Module, n.Error) + return n + } + for _, command := range d.Runs { + if _, err := b.Run(ctx, "sh", "-c", command); err != nil { + return fail(err) + } + } + if len(d.Paths) == 0 { + return fail(errors.New("it runs a dump and names no directory to keep it from")) + } + var missing []string + for _, p := range d.Paths { + if _, err := os.Stat(p); err != nil { + missing = append(missing, p) + } + } + if len(missing) > 0 { + return fail(fmt.Errorf("%s does not exist", strings.Join(missing, ", "))) + } + out, err := b.restic(ctx, append([]string{"backup", "--json", "--tag", tagOf(d.Module)}, d.Paths...)...) + if err != nil { + return fail(err) + } + scanner := bufio.NewScanner(strings.NewReader(out)) + scanner.Buffer(make([]byte, 1024*1024), 16*1024*1024) + for scanner.Scan() { + var m struct { + MessageType string `json:"message_type"` + SnapshotID string `json:"snapshot_id"` + } + if json.Unmarshal(scanner.Bytes(), &m) == nil && m.MessageType == "summary" && len(m.SnapshotID) >= 8 { + n.Snapshot = m.SnapshotID[:8] + } + } + n.OK = true + b.Say("%s: backed up (%s)", d.Module, n.Snapshot) + return n +} + +// BackUp is a night: every module, or one, then the rotation. It answers each module's outcome. +func (b *Backups) BackUp(ctx context.Context, only string) (map[string]Night, error) { + b.mu.Lock() + defer b.mu.Unlock() + if err := b.ensureRepository(ctx); err != nil { + return nil, err + } + all, err := b.Declared() + if err != nil { + return nil, err + } + chosen := all + if only != "" { + chosen = nil + var names []string + for _, d := range all { + names = append(names, d.Module) + if d.Module == only { + chosen = append(chosen, d) + } + } + if len(chosen) == 0 { + return nil, fmt.Errorf("%s declares nothing to back up on this machine; it backs up %s", only, orNothing(names)) + } + } + outcome := map[string]Night{} + for _, d := range chosen { + outcome[d.Module] = b.one(ctx, d) + b.record(d.Module, outcome[d.Module]) + } + if _, err := b.restic(ctx, "forget", "--prune", "--group-by", "host,tags", + "--keep-daily", fmt.Sprint(Keep.Daily), "--keep-weekly", fmt.Sprint(Keep.Weekly), "--keep-monthly", fmt.Sprint(Keep.Monthly)); err != nil { + b.Say("thinning the restore points failed, and every one is kept: %v", err) + } + return outcome, nil +} + +func orNothing(names []string) string { + if len(names) == 0 { + return "nothing" + } + return strings.Join(names, ", ") +} + +// Snapshots is the restore points, of one module or all. +func (b *Backups) Snapshots(ctx context.Context, module string) ([]Snapshot, error) { + args := []string{"snapshots", "--json"} + if module != "" { + args = append(args, "--tag", tagOf(module)) + } + out, err := b.restic(ctx, args...) + if err != nil { + return nil, err + } + var snaps []Snapshot + if strings.TrimSpace(out) == "" { + return nil, nil + } + if err := json.Unmarshal([]byte(out), &snaps); err != nil { + return nil, fmt.Errorf("restic listed its snapshots in a form this holder does not read: %v", err) + } + return snaps, nil +} + +// ModuleBackups is what `backed-up` says about one module. +type ModuleBackups struct { + Module string `json:"module"` + Runs int `json:"runs"` + Paths []string `json:"paths"` + LastNight *Night `json:"lastNight"` + RestorePoints int `json:"restorePoints"` + Newest *Snapshot `json:"newest,omitempty"` +} + +// BackedUp is what is backed up here: each module, what it declared, its last night and its restore +// points. +func (b *Backups) BackedUp(ctx context.Context, module string) ([]ModuleBackups, error) { + declared, err := b.Declared() + if err != nil { + return nil, err + } + nights := b.Nights() + snaps, _ := b.Snapshots(ctx, module) + out := []ModuleBackups{} + for _, d := range declared { + if module != "" && d.Module != module { + continue + } + m := ModuleBackups{Module: d.Module, Runs: len(d.Runs), Paths: d.Paths} + if n, ok := nights[d.Module]; ok { + m.LastNight = &n + } + for _, s := range snaps { + if slices.Contains(s.Tags, tagOf(d.Module)) { + m.RestorePoints++ + newest := s + m.Newest = &newest + } + } + out = append(out, m) + } + return out, nil +} + +// Restored is what a restore put where. +type Restored struct { + Module string `json:"module"` + From Snapshot `json:"from"` + Restored []string `json:"restored"` + Live string `json:"live"` +} + +// Restore puts a module's data from a restore point BESIDE the live data: each directory as +// .restored-. A target that already exists is refused, never overwritten. +func (b *Backups) Restore(ctx context.Context, module, snapshot, path string) (*Restored, error) { + b.mu.Lock() + defer b.mu.Unlock() + mine, err := b.Snapshots(ctx, module) + if err != nil { + return nil, err + } + if len(mine) == 0 { + return nil, fmt.Errorf("%s has no restore point on this machine", module) + } + chosen := mine[len(mine)-1] + if snapshot != "" { + found := false + var listed []string + for _, s := range mine { + listed = append(listed, fmt.Sprintf("%s (%s)", s.ShortID, s.Time.Format(time.RFC3339))) + if s.ShortID == snapshot || strings.HasPrefix(s.ID, snapshot) { + chosen, found = s, true + } + } + if !found { + return nil, fmt.Errorf("%s has no restore point %s; it has %s", module, snapshot, strings.Join(listed, ", ")) + } + } + paths := chosen.Paths + if path != "" { + if !slices.Contains(chosen.Paths, path) { + return nil, fmt.Errorf("restore point %s of %s holds %s, not %s", chosen.ShortID, module, strings.Join(chosen.Paths, ", "), path) + } + paths = []string{path} + } + stamp := b.Now().UTC().Format("20060102-150405") + r := &Restored{Module: module, From: chosen, Live: "untouched — swapping it in is a person's act"} + for _, p := range paths { + target := p + ".restored-" + stamp + if _, err := os.Stat(target); err == nil { + return nil, fmt.Errorf("%s already exists; nothing is restored over anything", target) + } + if _, err := b.restic(ctx, "restore", chosen.ID+":"+p, "--target", target); err != nil { + return nil, err + } + r.Restored = append(r.Restored, target) + b.Say("%s: restored %s from %s to %s", module, p, chosen.ShortID, target) + } + return r, nil +} + +// Check is the weekly look at the repository's own integrity, with a sample of the data read back. +func (b *Backups) Check(ctx context.Context) error { + b.mu.Lock() + defer b.mu.Unlock() + _, err := b.restic(ctx, "check", "--read-data-subset", "5%") + return err +} + +// nextNight is when the next night is due: the given hour, local time, today while it is still +// ahead, else tomorrow. +func nextNight(now time.Time, hour int) time.Time { + next := time.Date(now.Year(), now.Month(), now.Day(), hour, 0, 0, 0, now.Location()) + if !next.After(now) { + next = next.AddDate(0, 0, 1) + } + return next +} + +// missedANight is whether a night was missed: the newest good night of any module is older than a +// day and a bit — the machine was off, or this module was not running, at the hour. +func missedANight(nights map[string]Night, now time.Time) bool { + var newest time.Time + for _, n := range nights { + if n.OK && n.At.After(newest) { + newest = n.At + } + } + return newest.IsZero() || now.Sub(newest) > 26*time.Hour +} diff --git a/modules/restic/cmd/restic-backups/backups_test.go b/modules/restic/cmd/restic-backups/backups_test.go new file mode 100644 index 0000000..cf85a03 --- /dev/null +++ b/modules/restic/cmd/restic-backups/backups_test.go @@ -0,0 +1,228 @@ +package main + +// The machine's backups over a fake restic and fake stores (novox/hq ADR 0214, to-be 43), and — where +// restic is installed — over the real one, on throwaway directories. + +import ( + "context" + "encoding/json" + "errors" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "testing" + "time" +) + +const composed = "# What the modules on this machine back up, composed by the mesh. Do not edit.\n" + + "# postgres\nrun docker exec -u postgres postgres sh -c 'pg-dump-all'\npath /var/lib/mesh-store/dumps\n" + + "# mailu\npath /var/lib/mailu/data-mail\npath /var/lib/mailu/data-dkim\n" + +func placed(t *testing.T, declared string) Where { + t.Helper() + dir := t.TempDir() + must(t, os.WriteFile(filepath.Join(dir, "backups.conf"), []byte(declared), 0o600)) + must(t, os.WriteFile(filepath.Join(dir, "pw"), []byte("secret\n"), 0o600)) + return Where{Declared: filepath.Join(dir, "backups.conf"), Repository: filepath.Join(dir, "repo"), + PasswordFile: filepath.Join(dir, "pw"), State: dir} +} + +func must(t *testing.T, err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } +} + +func quiet(string, ...any) {} + +func TestTheComposedFileIsReadIntoEachModulesRunsAndPaths(t *testing.T) { + got, err := parseDeclared(composed) + must(t, err) + want := []Declared{ + {Module: "postgres", Runs: []string{"docker exec -u postgres postgres sh -c 'pg-dump-all'"}, Paths: []string{"/var/lib/mesh-store/dumps"}}, + {Module: "mailu", Paths: []string{"/var/lib/mailu/data-mail", "/var/lib/mailu/data-dkim"}}, + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("read %#v, want %#v", got, want) + } +} + +func TestALineThisHolderDoesNotReadIsRefusedNamingTheModule(t *testing.T) { + for _, bad := range []string{"# pg\ncopy /x\n", "# pg\npath relative/dir\n"} { + if _, err := parseDeclared(bad); err == nil || !strings.Contains(err.Error(), "pg contributes a backup line") { + t.Errorf("%q: %v", bad, err) + } + } +} + +func TestResticAndTheDumpsRunThroughSudoWhereTheAccountIsNotRoot(t *testing.T) { + if p, a := escalated(1000, "restic", []string{"snapshots"}); p != "sudo" || !reflect.DeepEqual(a, []string{"-n", "restic", "snapshots"}) { + t.Errorf("as an account: %s %v", p, a) + } + if p, a := escalated(0, "restic", []string{"snapshots"}); p != "restic" || !reflect.DeepEqual(a, []string{"snapshots"}) { + t.Errorf("as root: %s %v", p, a) + } +} + +func TestANightDumpsBeforeEachSnapshotAndOneFailingModuleFailsOnlyItself(t *testing.T) { + where := placed(t, "# pg\nrun dump-it\npath /\n# broken\nrun fail-it\npath /\n# mail\npath /\n") + var calls []string + run := func(_ context.Context, name string, args ...string) (string, error) { + line := name + " " + strings.Join(args, " ") + if name == "restic" { + line = "restic " + strings.Join(args[5:], " ") + } + calls = append(calls, line) + switch { + case line == "sh -c fail-it": + return "", errors.New("the dump failed") + case strings.HasPrefix(line, "restic backup"): + return "{\"message_type\":\"status\"}\n{\"message_type\":\"summary\",\"snapshot_id\":\"abcdef0123456789\"}\n", nil + } + return "", nil + } + b := &Backups{Where: where, Run: run, Now: func() time.Time { return time.Date(2026, 10, 6, 3, 0, 0, 0, time.UTC) }, Say: quiet} + outcome, err := b.BackUp(context.Background(), "") + must(t, err) + if !outcome["pg"].OK || outcome["pg"].Snapshot != "abcdef01" { + t.Errorf("pg: %+v", outcome["pg"]) + } + if outcome["broken"].OK || !strings.Contains(outcome["broken"].Error, "the dump failed") { + t.Errorf("broken: %+v", outcome["broken"]) + } + if !outcome["mail"].OK { + t.Errorf("mail failed with broken: %+v", outcome["mail"]) + } + want := []string{ + "restic cat config", + "sh -c dump-it", + "restic backup --json --tag module=pg /", + "sh -c fail-it", + "restic backup --json --tag module=mail /", + "restic forget --prune --group-by host,tags --keep-daily 14 --keep-weekly 8 --keep-monthly 6", + } + if !reflect.DeepEqual(calls, want) { + t.Errorf("ran\n%s\nwant\n%s", strings.Join(calls, "\n"), strings.Join(want, "\n")) + } + // Recorded, so `backed-up` and the missed-night check read it. + var nights map[string]Night + raw, err := os.ReadFile(filepath.Join(where.State, "nights.json")) + must(t, err) + must(t, json.Unmarshal(raw, &nights)) + if nights["broken"].OK || !nights["mail"].OK { + t.Errorf("recorded %+v", nights) + } +} + +func TestARepositoryThatWillNotOpenIsNeverReplaced(t *testing.T) { + var calls []string + run := func(_ context.Context, _ string, args ...string) (string, error) { + calls = append(calls, strings.Join(args[5:], " ")) + if args[5] == "cat" { + return "", errors.New("Fatal: wrong password or no key found") + } + return "", nil + } + b := &Backups{Where: placed(t, "# pg\npath /\n"), Run: run, Now: time.Now, Say: quiet} + if _, err := b.BackUp(context.Background(), ""); err == nil || !strings.Contains(err.Error(), "cannot be opened, and is left as it is") { + t.Fatalf("got %v", err) + } + for _, c := range calls { + if c == "init" { + t.Fatal("it made a new repository over one it could not open") + } + } +} + +func TestADeclaredDirectoryThatDoesNotExistFailsThatModulesNight(t *testing.T) { + b := &Backups{Where: placed(t, "# pg\npath /nowhere/at/all\n"), Run: func(context.Context, string, ...string) (string, error) { return "", nil }, + Now: time.Now, Say: quiet} + outcome, err := b.BackUp(context.Background(), "") + must(t, err) + if outcome["pg"].OK || !strings.Contains(outcome["pg"].Error, "/nowhere/at/all does not exist") { + t.Fatalf("pg: %+v", outcome["pg"]) + } +} + +func TestANightIsDueAtTheHourAndAMissedOneIsNoticed(t *testing.T) { + morning := time.Date(2026, 10, 6, 1, 30, 0, 0, time.Local) + if got := nextNight(morning, 3); !got.Equal(time.Date(2026, 10, 6, 3, 0, 0, 0, time.Local)) { + t.Errorf("from the morning: %v", got) + } + afternoon := time.Date(2026, 10, 6, 15, 0, 0, 0, time.Local) + if got := nextNight(afternoon, 3); !got.Equal(time.Date(2026, 10, 7, 3, 0, 0, 0, time.Local)) { + t.Errorf("from the afternoon: %v", got) + } + at := func(day int, ok bool) map[string]Night { + return map[string]Night{"pg": {OK: ok, At: time.Date(2026, 10, day, 3, 5, 0, 0, time.Local)}} + } + for _, c := range []struct { + nights map[string]Night + missed bool + }{{map[string]Night{}, true}, {at(6, true), false}, {at(4, true), true}, {at(6, false), true}} { + if got := missedANight(c.nights, afternoon); got != c.missed { + t.Errorf("%+v: missed %v", c.nights, got) + } + } +} + +// The real thing, where restic is installed: a dump, a snapshot, a mistake, and a restore beside. +func TestWithTheRealResticAMistakeIsUndoneBesideTheLiveData(t *testing.T) { + if _, err := exec.LookPath("restic"); err != nil { + t.Skip("restic is not installed") + } + root := t.TempDir() + store, dumps := filepath.Join(root, "store"), filepath.Join(root, "dumps") + must(t, os.Mkdir(store, 0o700)) + must(t, os.Mkdir(dumps, 0o700)) + must(t, os.WriteFile(filepath.Join(store, "mailbox"), []byte("the only copy of a letter\n"), 0o600)) + where := placed(t, "# mail\npath "+store+"\n# pg\nrun echo 'every row' > "+dumps+"/all.dump\npath "+dumps+"\n") + // As whoever runs the test, against its own repository: no sudo. + run := func(ctx context.Context, name string, args ...string) (string, error) { + out, err := exec.CommandContext(ctx, name, args...).Output() + if ee, ok := err.(*exec.ExitError); ok { + return string(out), errors.New(string(ee.Stderr)) + } + return string(out), err + } + b := &Backups{Where: where, Run: run, Now: func() time.Time { return time.Date(2026, 10, 6, 3, 0, 0, 0, time.UTC) }, Say: quiet} + ctx := context.Background() + + night, err := b.BackUp(ctx, "") + must(t, err) + if !night["mail"].OK || !night["pg"].OK { + t.Fatalf("the night: %+v", night) + } + if raw, _ := os.ReadFile(filepath.Join(dumps, "all.dump")); string(raw) != "every row\n" { + t.Fatalf("the dump: %q", raw) + } + + // The mistake. + must(t, os.Remove(filepath.Join(store, "mailbox"))) + + listed, err := b.BackedUp(ctx, "") + must(t, err) + if len(listed) != 2 || listed[0].RestorePoints != 1 || listed[1].RestorePoints != 1 { + t.Fatalf("listed %+v", listed) + } + + restored, err := b.Restore(ctx, "mail", "", "") + must(t, err) + if len(restored.Restored) != 1 || !strings.HasSuffix(restored.Restored[0], "store.restored-20261006-030000") { + t.Fatalf("restored %+v", restored) + } + if raw, _ := os.ReadFile(filepath.Join(restored.Restored[0], "mailbox")); string(raw) != "the only copy of a letter\n" { + t.Fatalf("the restored letter: %q", raw) + } + if _, err := os.Stat(filepath.Join(store, "mailbox")); err == nil { + t.Fatal("the restore wrote into the live directory") + } + + // Never over anything: the same restore again finds its target taken. + if _, err := b.Restore(ctx, "mail", "", ""); err == nil || !strings.Contains(err.Error(), "nothing is restored over anything") { + t.Fatalf("a second restore: %v", err) + } +} diff --git a/modules/restic/cmd/restic-backups/main.go b/modules/restic/cmd/restic-backups/main.go new file mode 100644 index 0000000..9dc24f9 --- /dev/null +++ b/modules/restic/cmd/restic-backups/main.go @@ -0,0 +1,137 @@ +// restic-backups (novox/hq ADR 0214, to-be 43): the machine's backups. One binary, launched by the +// machine's tool runtime and speaking MCP to it over stdio through the Go SDK (ADR 0193, ADR 0198). It +// serves the node-backup seat's three verbs — what is backed up, take one now, restore beside the +// live data — and, beside them, runs the night that happens without anyone asking. +// +// stdout is the MCP channel; everything this module says, it says on stderr. +package main + +import ( + "context" + "fmt" + "os" + "strconv" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// Seat is the role this module holds. +const Seat = "node-backup" + +func say(format string, args ...any) { + fmt.Fprintf(os.Stderr, "[restic] "+format+"\n", args...) +} + +func main() { + where, err := whereFromEnv() + if err != nil { + say("%v", err) + os.Exit(1) + } + b := &Backups{Where: where, Run: execRunner, Now: time.Now, Say: say} + go nights(b) + if err := stdio.Serve("", tools(b)); err != nil { + say("%v", err) + os.Exit(1) + } +} + +// nights runs at the hour, every module; and at start, if a night was missed — the machine was off +// or this module was not running at the hour — one soon rather than a day later. Not at once: a +// machine just started has its stores still coming up. +func nights(b *Backups) { + hour := 3 + if h, err := strconv.Atoi(os.Getenv("MESH_BACKUP_HOUR")); err == nil && h >= 0 && h < 24 { + hour = h + } + if missedANight(b.Nights(), time.Now()) { + time.Sleep(10 * time.Minute) + night(b) + } + for { + time.Sleep(time.Until(nextNight(time.Now(), hour))) + night(b) + } +} + +func night(b *Backups) { + ctx := context.Background() + outcome, err := b.BackUp(ctx, "") + if err != nil { + say("the night did not run: %v", err) + return + } + var failed []string + for module, n := range outcome { + if !n.OK { + failed = append(failed, module) + } + } + if len(failed) > 0 { + say("the night left %s without a backup", strings.Join(failed, ", ")) + } + // Sundays, the repository's own integrity with a sample of the data read back. + if time.Now().Weekday() == time.Sunday { + if err := b.Check(ctx); err != nil { + say("the repository does NOT check out: %v", err) + } else { + say("the repository checks out") + } + } +} + +// ---- the seat's verbs -------------------------------------------------------------------------- + +func str(description string) map[string]any { + return map[string]any{"type": "string", "description": description} +} + +func arg(a map[string]any, k string) string { + v, _ := a[k].(string) + return strings.TrimSpace(v) +} + +// verb is one of the seat's verbs: listed as `.`, so the runtime serves it on the seat's +// subject. +func verb(name, description string, input map[string]any, run func(a map[string]any) (any, error)) stdio.Tool { + return stdio.Tool{Name: Seat + "." + name, Description: description, Input: input, Run: run} +} + +func tools(b *Backups) []stdio.Tool { + return []stdio.Tool{ + verb("backed-up", "What this machine backs up: each module, what it declared, its last good night, how many restore points are kept.", + map[string]any{"module": str("one module (optional)")}, + func(a map[string]any) (any, error) { return b.BackedUp(context.Background(), arg(a, "module")) }), + verb("now", "Take a backup now, of one module or of every module on this machine — before a migration, a retirement or anything else that could go wrong. Answers when it has started; `backed-up` says how it went.", + map[string]any{"module": str("one module (optional)")}, + func(a map[string]any) (any, error) { + only := arg(a, "module") + // A night of a large store outlasts any call; it is started, and its outcome recorded. + go func() { + if _, err := b.BackUp(context.Background(), only); err != nil { + say("a backup asked for now failed: %v", err) + } + }() + started := only + if started == "" { + started = "every module on this machine" + } + return map[string]any{"started": started, "follow": Seat + ".backed-up"}, nil + }), + verb("restore", "Restore one module's data from a restore point BESIDE the live data, never over it: each directory as .restored-. Swapping it in is a person's act.", + map[string]any{ + "module": str("the module"), + "snapshot": str("the restore point (the newest when omitted)"), + "path": str("one of the module's directories (all of them when omitted)"), + }, + func(a map[string]any) (any, error) { + module := arg(a, "module") + if module == "" { + return nil, fmt.Errorf("module is required") + } + return b.Restore(context.Background(), module, arg(a, "snapshot"), arg(a, "path")) + }), + } +} diff --git a/modules/restic/go.mod b/modules/restic/go.mod new file mode 100644 index 0000000..f010f2f --- /dev/null +++ b/modules/restic/go.mod @@ -0,0 +1,5 @@ +module restic + +go 1.25.0 + +require git.novox.be/novox/mesh-sdk/go v0.1.7 diff --git a/modules/restic/go.sum b/modules/restic/go.sum new file mode 100644 index 0000000..b474419 --- /dev/null +++ b/modules/restic/go.sum @@ -0,0 +1,2 @@ +git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w= +git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= diff --git a/modules/restic/module.json b/modules/restic/module.json index f76a457..f304b89 100644 --- a/modules/restic/module.json +++ b/modules/restic/module.json @@ -45,12 +45,12 @@ { "name": "tools", "kind": "bundle", - "language": "typescript", - "entrypoints": [ - "tools/index.js" - ], + "language": "go", + "system": "arch", + "from": "cmd/restic-backups", + "binary": "restic-backups", "loads": [ - "tools/index.js" + "restic-backups" ], "env": { "MESH_BACKUP_DECLARED": "${dir:state}/backups.conf", diff --git a/modules/restic/package.json b/modules/restic/package.json deleted file mode 100644 index bdcffc0..0000000 --- a/modules/restic/package.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "@novox/module-restic", - "version": "0.1.0", - "description": "restic \u2014 the machine's backups: holds the node-backup seat, takes a nightly restore point of what every module on the machine declares, keeps 14 daily, 8 weekly and 6 monthly, and restores beside the live data (novox/hq ADR 0214, to-be 43).", - "type": "module", - "private": true, - "dependencies": { - "@novox/mesh-sdk": "^0.1.1" - }, - "devDependencies": { - "@types/node": "^22.0.0", - "typescript": "^5.6.0" - }, - "scripts": { - "build": "tsc client.ts tools/index.ts --module NodeNext --moduleResolution NodeNext --target ES2022 --rootDir . --outDir dist", - "test": "node --test --experimental-strip-types 'test/*.test.ts'" - } -} diff --git a/modules/restic/test/client.test.ts b/modules/restic/test/client.test.ts deleted file mode 100644 index 599dd2f..0000000 --- a/modules/restic/test/client.test.ts +++ /dev/null @@ -1,142 +0,0 @@ -// The machine's backups over a fake restic and fake stores (novox/hq ADR 0214, to-be 43), and — where -// restic is installed — over the real one, on throwaway directories. -import { test } from "node:test"; -import assert from "node:assert/strict"; -import { execFileSync } from "node:child_process"; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { Backups, escalated, missedANight, nextNight, parseDeclared, type Runner } from "../client.ts"; - -const COMPOSED = - "# What the modules on this machine back up, composed by the mesh. Do not edit.\n" + - "# postgres\nrun docker exec -u postgres postgres sh -c 'pg-dump-all'\npath /var/lib/mesh-store/dumps\n" + - "# mailu\npath /var/lib/mailu/data-mail\npath /var/lib/mailu/data-dkim\n"; - -function place(declared: string) { - const dir = mkdtempSync(join(tmpdir(), "restic-test-")); - writeFileSync(join(dir, "backups.conf"), declared); - writeFileSync(join(dir, "pw"), "secret\n"); - return { declared: join(dir, "backups.conf"), repository: join(dir, "repo"), passwordFile: join(dir, "pw"), state: dir }; -} - -test("the composed file is read into each module's runs and paths, the header comment being nothing", () => { - assert.deepEqual(parseDeclared(COMPOSED), [ - { module: "postgres", runs: ["docker exec -u postgres postgres sh -c 'pg-dump-all'"], paths: ["/var/lib/mesh-store/dumps"] }, - { module: "mailu", runs: [], paths: ["/var/lib/mailu/data-mail", "/var/lib/mailu/data-dkim"] }, - ]); -}); - -test("a line this holder does not read is refused, naming the module, rather than skipped", () => { - assert.throws(() => parseDeclared("# pg\ncopy /x\n"), /pg contributes a backup line this holder does not read: copy \/x/); - assert.throws(() => parseDeclared("# pg\npath relative/dir\n"), /pg contributes/); -}); - -test("restic and the dumps run through sudo where the account is not root", () => { - assert.deepEqual(escalated("restic", ["snapshots"], 1000), ["sudo", ["-n", "restic", "snapshots"]]); - assert.deepEqual(escalated("restic", ["snapshots"], 0), ["restic", ["snapshots"]]); -}); - -test("a night runs each module's dump before its snapshot, and one failing module fails only itself", async () => { - const where = place("# pg\nrun dump-it\npath /\n# broken\nrun fail-it\npath /\n# mail\npath /\n"); - const calls: string[] = []; - const run: Runner = async (cmd, args) => { - const line = cmd === "restic" ? `restic ${args.slice(5).join(" ")}` : `${cmd} ${args.join(" ")}`; - calls.push(line); - if (line === "sh -c fail-it") throw new Error("the dump failed"); - if (line.startsWith("restic backup")) return '{"message_type":"status"}\n{"message_type":"summary","snapshot_id":"abcdef0123456789"}\n'; - return ""; - }; - const outcome = await new Backups(where, run, () => new Date("2026-10-06T03:00:00Z"), () => {}).backUp(); - assert.equal(outcome.pg.ok, true); - assert.equal(outcome.pg.snapshot, "abcdef01"); - assert.equal(outcome.broken.ok, false); - assert.match(outcome.broken.error ?? "", /the dump failed/); - assert.equal(outcome.mail.ok, true); - assert.deepEqual(calls, [ - "restic cat config", - "sh -c dump-it", - "restic backup --json --tag module=pg /", - "sh -c fail-it", - "restic backup --json --tag module=mail /", - "restic forget --prune --group-by host,tags --keep-daily 14 --keep-weekly 8 --keep-monthly 6", - ]); - // Recorded, so `backed-up` and the missed-night check read it. - const nights = JSON.parse(readFileSync(join(where.state, "nights.json"), "utf8")); - assert.equal(nights.broken.ok, false); - assert.equal(nights.mail.ok, true); -}); - -test("a repository that exists and will not open is never replaced", async () => { - const where = place("# pg\npath /\n"); - const calls: string[] = []; - const run: Runner = async (cmd, args) => { - calls.push(args.slice(5).join(" ")); - if (args.includes("cat")) throw new Error("Fatal: wrong password or no key found"); - return ""; - }; - await assert.rejects(new Backups(where, run, undefined, () => {}).backUp(), /cannot be opened, and is left as it is/); - assert.ok(!calls.includes("init"), "it made a new repository over one it could not open"); -}); - -test("a declared directory that does not exist fails that module's night", async () => { - const where = place("# pg\npath /nowhere/at/all\n"); - const run: Runner = async () => ""; - const outcome = await new Backups(where, run, undefined, () => {}).backUp(); - assert.equal(outcome.pg.ok, false); - assert.match(outcome.pg.error ?? "", /\/nowhere\/at\/all does not exist/); -}); - -test("a night is due at the hour today while it is ahead, else tomorrow; a missed one is noticed", () => { - const morning = new Date(2026, 9, 6, 1, 30); - assert.equal(nextNight(morning, 3).getTime(), new Date(2026, 9, 6, 3, 0).getTime()); - const afternoon = new Date(2026, 9, 6, 15, 0); - assert.equal(nextNight(afternoon, 3).getTime(), new Date(2026, 9, 7, 3, 0).getTime()); - assert.equal(missedANight({}, afternoon), true); - assert.equal(missedANight({ pg: { ok: true, at: new Date(2026, 9, 6, 3, 5).toISOString() } }, afternoon), false); - assert.equal(missedANight({ pg: { ok: true, at: new Date(2026, 9, 4, 3, 5).toISOString() } }, afternoon), true); - assert.equal(missedANight({ pg: { ok: false, at: new Date(2026, 9, 6, 3, 5).toISOString() } }, afternoon), true); -}); - -// The real thing, where restic is installed: a dump, a snapshot, a mistake, and a restore beside. -const hasRestic = (() => { - try { - execFileSync("restic", ["version"], { stdio: "ignore" }); - return true; - } catch { - return false; - } -})(); - -test("with the real restic: a mistake is undone by a restore beside the live data", { skip: !hasRestic && "restic is not installed" }, async () => { - const root = mkdtempSync(join(tmpdir(), "restic-real-")); - const store = join(root, "store"); - const dumps = join(root, "dumps"); - mkdirSync(store); - mkdirSync(dumps); - writeFileSync(join(store, "mailbox"), "the only copy of a letter\n"); - const where = place(`# mail\npath ${store}\n# pg\nrun echo 'every row' > ${dumps}/all.dump\npath ${dumps}\n`); - const backups = new Backups(where, undefined, () => new Date("2026-10-06T03:00:00Z"), () => {}); - // escalated() would sudo; the test runs as whoever it is, against its own repository. - (backups as unknown as { run: Runner }).run = async (cmd, args) => execFileSync(cmd, args, { encoding: "utf8" }); - - const night = await backups.backUp(); - assert.equal(night.mail.ok, true, night.mail.error); - assert.equal(night.pg.ok, true, night.pg.error); - assert.equal(readFileSync(join(dumps, "all.dump"), "utf8"), "every row\n"); - - // The mistake. - rmSync(join(store, "mailbox")); - - const listed = await backups.backedUp(); - assert.deepEqual(listed.map((m) => [m.module, m.restorePoints]), [["mail", 1], ["pg", 1]]); - - const restored = await backups.restore("mail"); - assert.equal(restored.restored.length, 1); - assert.match(restored.restored[0], /store\.restored-20261006-030000$/); - assert.equal(readFileSync(join(restored.restored[0], "mailbox"), "utf8"), "the only copy of a letter\n"); - assert.ok(!existsSync(join(store, "mailbox")), "the restore wrote into the live directory"); - - // Never over anything: the same restore again finds its target taken. - await assert.rejects(backups.restore("mail"), /already exists; nothing is restored over anything/); -}); diff --git a/modules/restic/tools/index.ts b/modules/restic/tools/index.ts deleted file mode 100644 index 95f15c1..0000000 --- a/modules/restic/tools/index.ts +++ /dev/null @@ -1,78 +0,0 @@ -// The machine's backups: the node-backup seat's three verbs — what is backed up, take one now, -// restore beside the live data — and the night that runs without anyone asking (novox/hq ADR 0214, -// to-be 43). What is backed up is composed by the mesh from the modules the machine runs; this code -// only runs it. - -import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools"; -import { Backups, missedANight, nextNight, whereFromEnv } from "../client.js"; - -export function getSeatVerbs(backups: Backups): ToolDefinition[] { - return [ - { - name: "backed-up", - description: - "What this machine backs up: each module, what it declared, its last good night, how many restore points are kept.", - input: { module: { type: "string", description: "one module (optional)" } }, - run: async (args) => backups.backedUp(args.module ? String(args.module) : undefined), - }, - { - name: "now", - description: - "Take a backup now, of one module or of every module on this machine — before a migration, a retirement or anything else that could go wrong. Answers when it has started; `backed-up` says how it went.", - input: { module: { type: "string", description: "one module (optional)" } }, - run: async (args) => { - const only = args.module ? String(args.module) : undefined; - // A night of a large store outlasts any call; it is started, and its outcome recorded. - backups.backUp(only).catch((err) => console.error(`[restic] a backup asked for now failed: ${(err as Error).message}`)); - return { started: only ?? "every module on this machine", follow: "backed-up" }; - }, - }, - { - name: "restore", - description: - "Restore one module's data from a restore point BESIDE the live data, never over it: each directory as .restored-. Swapping it in is a person's act.", - input: { - module: { type: "string", description: "the module" }, - snapshot: { type: "string", description: "the restore point (the newest when omitted)" }, - path: { type: "string", description: "one of the module's directories (all of them when omitted)" }, - }, - run: async (args) => - backups.restore(String(args.module ?? ""), args.snapshot ? String(args.snapshot) : undefined, args.path ? String(args.path) : undefined), - }, - ]; -} - -const backups = new Backups(whereFromEnv()); -registerModuleTools("node-backup", () => getSeatVerbs(backups)); - -// The night. At the hour, every module; and at start, if a night was missed — the machine was off -// or this module was not running at the hour — one now rather than a day later. -const HOUR = Number(process.env.MESH_BACKUP_HOUR ?? "3"); - -async function night(): Promise { - try { - const outcome = await backups.backUp(); - const failed = Object.entries(outcome).filter(([, n]) => !n.ok).map(([m]) => m); - if (failed.length > 0) console.error(`[restic] the night left ${failed.join(", ")} without a backup`); - // Sundays, the repository's own integrity with a sample of the data read back. - if (new Date().getDay() === 0) { - await backups.check().then( - () => console.error("[restic] the repository checks out"), - (err) => console.error(`[restic] the repository does NOT check out: ${(err as Error).message}`), - ); - } - } catch (err) { - console.error(`[restic] the night did not run: ${(err as Error).message}`); - } -} - -function schedule(): void { - const at = nextNight(new Date(), HOUR); - setTimeout(() => void night().finally(schedule), at.getTime() - Date.now()); -} - -if (missedANight(backups.nights(), new Date())) { - // Not at once: a machine just started has its stores still coming up. - setTimeout(() => void night(), 10 * 60 * 1000); -} -schedule(); diff --git a/modules/restic/tsconfig.json b/modules/restic/tsconfig.json deleted file mode 100644 index 1f1b70a..0000000 --- a/modules/restic/tsconfig.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2022", - "module": "NodeNext", - "moduleResolution": "NodeNext", - "strict": true, - "esModuleInterop": true, - "skipLibCheck": true, - "noEmit": true - }, - "include": [ - "client.ts", - "tools/index.ts" - ] -}