// restic's own code, in the module (novox/hq ADR 0039): the machine's backups (ADR 0214, to-be 43). // // The mesh composes what to back up: every module on the machine contributes `backup` lines to the // node-backup seat, and the mesh writes them, each module's under a `# ` line and with its // directories already filled, into one file this module reads. Two kinds of line: // // run run as root before the module's snapshot — a consistent dump of a store // path a directory the module's snapshot keeps // // Each module gets one snapshot a night, tagged with its name, so a module is listed, kept and // restored on its own. Everything lands in one repository on the machine — deduplicated, so every // night is a complete restore point and only what changed costs space — and is thinned to 14 daily, // 8 weekly and 6 monthly. Against mistakes, not disasters: nothing leaves the machine. // // Root's: the dumps read every store and the repository holds every module's data, and the runtime // loading this bundle runs as the operator's account, so restic and the run lines go through sudo // without a prompt where the account is not root — fail2ban's way (ADR 0175 §4). import { execFile } from "node:child_process"; import { existsSync, readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { promisify } from "node:util"; const execFileP = promisify(execFile); /** A command runner, so the backups can be tested without restic or a store. */ export type Runner = (cmd: string, args: string[]) => Promise; /** The command as it is run: as given when this process is root, else through sudo without a prompt. */ export function escalated(cmd: string, args: string[], uid: number | undefined = process.getuid?.()): [string, string[]] { if (uid === 0) return [cmd, args]; return ["sudo", ["-n", cmd, ...args]]; } export const execRunner: Runner = async (cmd, args) => { const [program, argv] = escalated(cmd, args); try { // A night's dump of a large store takes a while; six hours is a dump that will not finish. const { stdout } = await execFileP(program, argv, { maxBuffer: 256 * 1024 * 1024, timeout: 6 * 3600 * 1000 }); return stdout; } catch (err) { const e = err as { code?: string | number; stderr?: string; stdout?: string; message?: string }; const said = `${e.stderr ?? ""}`.trim() || `${e.stdout ?? ""}`.trim(); if (program === "sudo" && /^sudo:/m.test(said)) { throw new Error(`${cmd} needs root and the runtime's account may not run it without a prompt: ${said}`); } const lines = said.split("\n").map((l) => l.trim()).filter(Boolean); throw new Error(lines.length ? lines.slice(-3).join(" / ") : (e.message ?? `${cmd} failed`)); } }; /** What one module declared. */ export interface Declared { module: string; runs: string[]; paths: string[]; } /** The composed file, read into each module's declaration, in the order the mesh wrote them. A line * before any module, a comment that is not a module's name, or a blank, is nothing. */ export function parseDeclared(text: string): Declared[] { const out: Declared[] = []; let current: Declared | undefined; for (const raw of text.split("\n")) { const line = raw.trim(); if (line === "") continue; const header = /^#\s*([a-z0-9][a-z0-9-]*)$/.exec(line); if (header) { current = { module: header[1], runs: [], paths: [] }; out.push(current); continue; } if (line.startsWith("#") || !current) continue; const [kind, ...rest] = line.split(/\s+/); const value = line.slice(kind.length).trim(); if (kind === "run" && value) current.runs.push(value); else if (kind === "path" && rest.length === 1 && value.startsWith("/")) current.paths.push(value); else throw new Error(`${current.module} contributes a backup line this holder does not read: ${line}`); } return out.filter((d) => d.runs.length > 0 || d.paths.length > 0); } /** One restore point, as restic lists it. */ export interface Snapshot { id: string; short_id: string; time: string; paths: string[]; tags?: string[]; hostname: string; } /** How one module's last night went. */ export interface Night { ok: boolean; at: string; snapshot?: string; error?: string; } export const KEEP = { daily: 14, weekly: 8, monthly: 6 }; export function tagOf(module: string): string { return `module=${module}`; } export interface Where { declared: string; repository: string; passwordFile: string; state: string; } export function whereFromEnv(env: NodeJS.ProcessEnv = process.env): Where { const need = (k: string) => { const v = env[k]; if (!v) throw new Error(`${k} is not set; the mesh gives it to this module's tools`); return v; }; return { declared: need("MESH_BACKUP_DECLARED"), repository: need("MESH_BACKUP_REPOSITORY"), passwordFile: need("MESH_BACKUP_PASSWORD_FILE"), state: need("MESH_BACKUP_STATE"), }; } export class Backups { private busy: Promise = Promise.resolve(); readonly where: Where; private run: Runner; private readonly now: () => Date; private readonly say: (line: string) => void; constructor( where: Where, run: Runner = execRunner, now: () => Date = () => new Date(), say: (line: string) => void = (l) => console.error(`[restic] ${l}`), ) { this.where = where; this.run = run; this.now = now; this.say = say; } private restic(args: string[]): Promise { return this.run("restic", ["--repo", this.where.repository, "--password-file", this.where.passwordFile, "--no-cache", ...args]); } /** One thing at a time: two nights, or a night and a restore, never share a dump. */ private serial(work: () => Promise): Promise { const next = this.busy.then(work, work); this.busy = next.catch(() => undefined); return next; } declared(): Declared[] { return parseDeclared(readFileSync(this.where.declared, "utf8")); } private nightsFile(): string { return join(this.where.state, "nights.json"); } nights(): Record { try { return JSON.parse(readFileSync(this.nightsFile(), "utf8")) as Record; } catch { return {}; } } private record(module: string, night: Night): void { const all = this.nights(); all[module] = night; writeFileSync(this.nightsFile(), JSON.stringify(all, null, 2) + "\n", { mode: 0o600 }); } /** The repository, made the first time. A repository that exists and cannot be opened is said, * never replaced: replacing it would discard every restore point to fix a password. */ async ensureRepository(): Promise { try { await this.restic(["cat", "config"]); } catch (err) { const why = String((err as Error).message); if (!/does not exist|unable to open config file|Is there a repository at the following location/i.test(why)) { throw new Error(`the repository at ${this.where.repository} cannot be opened, and is left as it is: ${why}`); } this.say(`no repository at ${this.where.repository}; making one`); await this.restic(["init"]); } } /** One module's night: its run lines, then one snapshot of its paths. A failure is that module's. */ private async one(d: Declared): Promise { const at = this.now().toISOString(); try { for (const command of d.runs) { await this.run("sh", ["-c", command]); } const missing = d.paths.filter((p) => !existsSync(p)); if (missing.length > 0) throw new Error(`${missing.join(", ")} does not exist`); if (d.paths.length === 0) throw new Error("it runs a dump and names no directory to keep it from"); const out = await this.restic(["backup", "--json", "--tag", tagOf(d.module), ...d.paths]); const summary = out .split("\n") .map((l) => { try { return JSON.parse(l) as { message_type?: string; snapshot_id?: string }; } catch { return {}; } }) .find((m) => m.message_type === "summary"); const night: Night = { ok: true, at, snapshot: summary?.snapshot_id?.slice(0, 8) }; this.say(`${d.module}: backed up (${night.snapshot ?? "no snapshot id reported"})`); return night; } catch (err) { const night: Night = { ok: false, at, error: String((err as Error).message) }; this.say(`${d.module}: NOT backed up: ${night.error}`); return night; } } /** A night: every module, or one, then the rotation. Returns each module's outcome. */ backUp(only?: string): Promise> { return this.serial(async () => { await this.ensureRepository(); const all = this.declared(); const chosen = only ? all.filter((d) => d.module === only) : all; if (only && chosen.length === 0) { throw new Error(`${only} declares nothing to back up on this machine; it backs up ${all.map((d) => d.module).join(", ") || "nothing"}`); } const outcome: Record = {}; for (const d of chosen) { outcome[d.module] = await this.one(d); this.record(d.module, outcome[d.module]); } await this.restic([ "forget", "--prune", "--group-by", "host,tags", "--keep-daily", String(KEEP.daily), "--keep-weekly", String(KEEP.weekly), "--keep-monthly", String(KEEP.monthly), ]).catch((err) => this.say(`thinning the restore points failed, and every one is kept: ${(err as Error).message}`)); return outcome; }); } async snapshots(module?: string): Promise { const args = ["snapshots", "--json"]; if (module) args.push("--tag", tagOf(module)); return JSON.parse((await this.restic(args)) || "[]") as Snapshot[]; } /** What is backed up here: each module, what it declared, its last night and its restore points. */ async backedUp(module?: string) { const nights = this.nights(); const snaps = await this.snapshots(module).catch(() => [] as Snapshot[]); return this.declared() .filter((d) => !module || d.module === module) .map((d) => { const mine = snaps.filter((s) => (s.tags ?? []).includes(tagOf(d.module))); return { module: d.module, runs: d.runs.length, paths: d.paths, lastNight: nights[d.module] ?? null, restorePoints: mine.length, newest: mine.length ? { snapshot: mine[mine.length - 1].short_id, at: mine[mine.length - 1].time } : null, }; }); } /** A module's data from a restore point, BESIDE the live data: each directory as * .restored-. A target that already exists is refused, never overwritten. */ restore(module: string, snapshot?: string, path?: string) { return this.serial(async () => { const mine = await this.snapshots(module); if (mine.length === 0) throw new Error(`${module} has no restore point on this machine`); const chosen = snapshot ? mine.find((s) => s.id.startsWith(snapshot) || s.short_id === snapshot) : mine[mine.length - 1]; if (!chosen) { throw new Error(`${module} has no restore point ${snapshot}; it has ${mine.map((s) => `${s.short_id} (${s.time})`).join(", ")}`); } const paths = path ? chosen.paths.filter((p) => p === path) : chosen.paths; if (paths.length === 0) throw new Error(`restore point ${chosen.short_id} of ${module} holds ${chosen.paths.join(", ")}, not ${path}`); const stamp = this.now().toISOString().replace(/[-:]/g, "").replace("T", "-").slice(0, 15); const restored: string[] = []; for (const p of paths) { const target = `${p}.restored-${stamp}`; if (existsSync(target)) throw new Error(`${target} already exists; nothing is restored over anything`); await this.restic(["restore", `${chosen.id}:${p}`, "--target", target]); restored.push(target); this.say(`${module}: restored ${p} from ${chosen.short_id} to ${target}`); } return { module, from: { snapshot: chosen.short_id, at: chosen.time }, restored, live: "untouched — swapping it in is a person's act" }; }); } /** The weekly look at the repository's own integrity, with a sample of the data read back. */ check(): Promise { return this.serial(() => this.restic(["check", "--read-data-subset", "5%"])); } } /** When the next night is due: the given hour, local time, today if it is still ahead, else tomorrow. */ export function nextNight(now: Date, hour: number): Date { const next = new Date(now); next.setHours(hour, 0, 0, 0); if (next.getTime() <= now.getTime()) next.setDate(next.getDate() + 1); return next; } /** Whether a night was missed: the newest good night of any module is older than a day and a bit — * the machine was off, or this module was not running, at the hour. */ export function missedANight(nights: Record, now: Date): boolean { const good = Object.values(nights).filter((n) => n.ok).map((n) => Date.parse(n.at)); if (good.length === 0) return true; return now.getTime() - Math.max(...good) > 26 * 3600 * 1000; }