jschoubben
32cd92baeb
ADR 0214 / to-be 43. restic keeps one repository per machine and takes a nightly snapshot per module — 14 daily, 8 weekly, 6 monthly — and restores beside the live data, never over it. postgres, mssql and mongodb contribute a consistent dump; minio, influxdb, the vault, mailu, gitea and nextcloud the directories that hold their data.
313 lines
13 KiB
TypeScript
313 lines
13 KiB
TypeScript
// restic's own code, in the module (novox/hq ADR 0039): the machine's backups (ADR 0214, to-be 43).
|
|
//
|
|
// The mesh composes what to back up: every module on the machine contributes `backup` lines to the
|
|
// node-backup seat, and the mesh writes them, each module's under a `# <module>` line and with its
|
|
// directories already filled, into one file this module reads. Two kinds of line:
|
|
//
|
|
// run <shell command> run as root before the module's snapshot — a consistent dump of a store
|
|
// path <directory> a directory the module's snapshot keeps
|
|
//
|
|
// Each module gets one snapshot a night, tagged with its name, so a module is listed, kept and
|
|
// restored on its own. Everything lands in one repository on the machine — deduplicated, so every
|
|
// night is a complete restore point and only what changed costs space — and is thinned to 14 daily,
|
|
// 8 weekly and 6 monthly. Against mistakes, not disasters: nothing leaves the machine.
|
|
//
|
|
// Root's: the dumps read every store and the repository holds every module's data, and the runtime
|
|
// loading this bundle runs as the operator's account, so restic and the run lines go through sudo
|
|
// without a prompt where the account is not root — fail2ban's way (ADR 0175 §4).
|
|
|
|
import { execFile } from "node:child_process";
|
|
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
|
import { join } from "node:path";
|
|
import { promisify } from "node:util";
|
|
|
|
const execFileP = promisify(execFile);
|
|
|
|
/** A command runner, so the backups can be tested without restic or a store. */
|
|
export type Runner = (cmd: string, args: string[]) => Promise<string>;
|
|
|
|
/** The command as it is run: as given when this process is root, else through sudo without a prompt. */
|
|
export function escalated(cmd: string, args: string[], uid: number | undefined = process.getuid?.()): [string, string[]] {
|
|
if (uid === 0) return [cmd, args];
|
|
return ["sudo", ["-n", cmd, ...args]];
|
|
}
|
|
|
|
export const execRunner: Runner = async (cmd, args) => {
|
|
const [program, argv] = escalated(cmd, args);
|
|
try {
|
|
// A night's dump of a large store takes a while; six hours is a dump that will not finish.
|
|
const { stdout } = await execFileP(program, argv, { maxBuffer: 256 * 1024 * 1024, timeout: 6 * 3600 * 1000 });
|
|
return stdout;
|
|
} catch (err) {
|
|
const e = err as { code?: string | number; stderr?: string; stdout?: string; message?: string };
|
|
const said = `${e.stderr ?? ""}`.trim() || `${e.stdout ?? ""}`.trim();
|
|
if (program === "sudo" && /^sudo:/m.test(said)) {
|
|
throw new Error(`${cmd} needs root and the runtime's account may not run it without a prompt: ${said}`);
|
|
}
|
|
const lines = said.split("\n").map((l) => l.trim()).filter(Boolean);
|
|
throw new Error(lines.length ? lines.slice(-3).join(" / ") : (e.message ?? `${cmd} failed`));
|
|
}
|
|
};
|
|
|
|
/** What one module declared. */
|
|
export interface Declared {
|
|
module: string;
|
|
runs: string[];
|
|
paths: string[];
|
|
}
|
|
|
|
/** The composed file, read into each module's declaration, in the order the mesh wrote them. A line
|
|
* before any module, a comment that is not a module's name, or a blank, is nothing. */
|
|
export function parseDeclared(text: string): Declared[] {
|
|
const out: Declared[] = [];
|
|
let current: Declared | undefined;
|
|
for (const raw of text.split("\n")) {
|
|
const line = raw.trim();
|
|
if (line === "") continue;
|
|
const header = /^#\s*([a-z0-9][a-z0-9-]*)$/.exec(line);
|
|
if (header) {
|
|
current = { module: header[1], runs: [], paths: [] };
|
|
out.push(current);
|
|
continue;
|
|
}
|
|
if (line.startsWith("#") || !current) continue;
|
|
const [kind, ...rest] = line.split(/\s+/);
|
|
const value = line.slice(kind.length).trim();
|
|
if (kind === "run" && value) current.runs.push(value);
|
|
else if (kind === "path" && rest.length === 1 && value.startsWith("/")) current.paths.push(value);
|
|
else throw new Error(`${current.module} contributes a backup line this holder does not read: ${line}`);
|
|
}
|
|
return out.filter((d) => d.runs.length > 0 || d.paths.length > 0);
|
|
}
|
|
|
|
/** One restore point, as restic lists it. */
|
|
export interface Snapshot {
|
|
id: string;
|
|
short_id: string;
|
|
time: string;
|
|
paths: string[];
|
|
tags?: string[];
|
|
hostname: string;
|
|
}
|
|
|
|
/** How one module's last night went. */
|
|
export interface Night {
|
|
ok: boolean;
|
|
at: string;
|
|
snapshot?: string;
|
|
error?: string;
|
|
}
|
|
|
|
export const KEEP = { daily: 14, weekly: 8, monthly: 6 };
|
|
|
|
export function tagOf(module: string): string {
|
|
return `module=${module}`;
|
|
}
|
|
|
|
export interface Where {
|
|
declared: string;
|
|
repository: string;
|
|
passwordFile: string;
|
|
state: string;
|
|
}
|
|
|
|
export function whereFromEnv(env: NodeJS.ProcessEnv = process.env): Where {
|
|
const need = (k: string) => {
|
|
const v = env[k];
|
|
if (!v) throw new Error(`${k} is not set; the mesh gives it to this module's tools`);
|
|
return v;
|
|
};
|
|
return {
|
|
declared: need("MESH_BACKUP_DECLARED"),
|
|
repository: need("MESH_BACKUP_REPOSITORY"),
|
|
passwordFile: need("MESH_BACKUP_PASSWORD_FILE"),
|
|
state: need("MESH_BACKUP_STATE"),
|
|
};
|
|
}
|
|
|
|
export class Backups {
|
|
private busy: Promise<unknown> = Promise.resolve();
|
|
readonly where: Where;
|
|
private run: Runner;
|
|
private readonly now: () => Date;
|
|
private readonly say: (line: string) => void;
|
|
|
|
constructor(
|
|
where: Where,
|
|
run: Runner = execRunner,
|
|
now: () => Date = () => new Date(),
|
|
say: (line: string) => void = (l) => console.error(`[restic] ${l}`),
|
|
) {
|
|
this.where = where;
|
|
this.run = run;
|
|
this.now = now;
|
|
this.say = say;
|
|
}
|
|
|
|
private restic(args: string[]): Promise<string> {
|
|
return this.run("restic", ["--repo", this.where.repository, "--password-file", this.where.passwordFile, "--no-cache", ...args]);
|
|
}
|
|
|
|
/** One thing at a time: two nights, or a night and a restore, never share a dump. */
|
|
private serial<T>(work: () => Promise<T>): Promise<T> {
|
|
const next = this.busy.then(work, work);
|
|
this.busy = next.catch(() => undefined);
|
|
return next;
|
|
}
|
|
|
|
declared(): Declared[] {
|
|
return parseDeclared(readFileSync(this.where.declared, "utf8"));
|
|
}
|
|
|
|
private nightsFile(): string {
|
|
return join(this.where.state, "nights.json");
|
|
}
|
|
|
|
nights(): Record<string, Night> {
|
|
try {
|
|
return JSON.parse(readFileSync(this.nightsFile(), "utf8")) as Record<string, Night>;
|
|
} catch {
|
|
return {};
|
|
}
|
|
}
|
|
|
|
private record(module: string, night: Night): void {
|
|
const all = this.nights();
|
|
all[module] = night;
|
|
writeFileSync(this.nightsFile(), JSON.stringify(all, null, 2) + "\n", { mode: 0o600 });
|
|
}
|
|
|
|
/** The repository, made the first time. A repository that exists and cannot be opened is said,
|
|
* never replaced: replacing it would discard every restore point to fix a password. */
|
|
async ensureRepository(): Promise<void> {
|
|
try {
|
|
await this.restic(["cat", "config"]);
|
|
} catch (err) {
|
|
const why = String((err as Error).message);
|
|
if (!/does not exist|unable to open config file|Is there a repository at the following location/i.test(why)) {
|
|
throw new Error(`the repository at ${this.where.repository} cannot be opened, and is left as it is: ${why}`);
|
|
}
|
|
this.say(`no repository at ${this.where.repository}; making one`);
|
|
await this.restic(["init"]);
|
|
}
|
|
}
|
|
|
|
/** One module's night: its run lines, then one snapshot of its paths. A failure is that module's. */
|
|
private async one(d: Declared): Promise<Night> {
|
|
const at = this.now().toISOString();
|
|
try {
|
|
for (const command of d.runs) {
|
|
await this.run("sh", ["-c", command]);
|
|
}
|
|
const missing = d.paths.filter((p) => !existsSync(p));
|
|
if (missing.length > 0) throw new Error(`${missing.join(", ")} does not exist`);
|
|
if (d.paths.length === 0) throw new Error("it runs a dump and names no directory to keep it from");
|
|
const out = await this.restic(["backup", "--json", "--tag", tagOf(d.module), ...d.paths]);
|
|
const summary = out
|
|
.split("\n")
|
|
.map((l) => { try { return JSON.parse(l) as { message_type?: string; snapshot_id?: string }; } catch { return {}; } })
|
|
.find((m) => m.message_type === "summary");
|
|
const night: Night = { ok: true, at, snapshot: summary?.snapshot_id?.slice(0, 8) };
|
|
this.say(`${d.module}: backed up (${night.snapshot ?? "no snapshot id reported"})`);
|
|
return night;
|
|
} catch (err) {
|
|
const night: Night = { ok: false, at, error: String((err as Error).message) };
|
|
this.say(`${d.module}: NOT backed up: ${night.error}`);
|
|
return night;
|
|
}
|
|
}
|
|
|
|
/** A night: every module, or one, then the rotation. Returns each module's outcome. */
|
|
backUp(only?: string): Promise<Record<string, Night>> {
|
|
return this.serial(async () => {
|
|
await this.ensureRepository();
|
|
const all = this.declared();
|
|
const chosen = only ? all.filter((d) => d.module === only) : all;
|
|
if (only && chosen.length === 0) {
|
|
throw new Error(`${only} declares nothing to back up on this machine; it backs up ${all.map((d) => d.module).join(", ") || "nothing"}`);
|
|
}
|
|
const outcome: Record<string, Night> = {};
|
|
for (const d of chosen) {
|
|
outcome[d.module] = await this.one(d);
|
|
this.record(d.module, outcome[d.module]);
|
|
}
|
|
await this.restic([
|
|
"forget", "--prune", "--group-by", "host,tags",
|
|
"--keep-daily", String(KEEP.daily), "--keep-weekly", String(KEEP.weekly), "--keep-monthly", String(KEEP.monthly),
|
|
]).catch((err) => this.say(`thinning the restore points failed, and every one is kept: ${(err as Error).message}`));
|
|
return outcome;
|
|
});
|
|
}
|
|
|
|
async snapshots(module?: string): Promise<Snapshot[]> {
|
|
const args = ["snapshots", "--json"];
|
|
if (module) args.push("--tag", tagOf(module));
|
|
return JSON.parse((await this.restic(args)) || "[]") as Snapshot[];
|
|
}
|
|
|
|
/** What is backed up here: each module, what it declared, its last night and its restore points. */
|
|
async backedUp(module?: string) {
|
|
const nights = this.nights();
|
|
const snaps = await this.snapshots(module).catch(() => [] as Snapshot[]);
|
|
return this.declared()
|
|
.filter((d) => !module || d.module === module)
|
|
.map((d) => {
|
|
const mine = snaps.filter((s) => (s.tags ?? []).includes(tagOf(d.module)));
|
|
return {
|
|
module: d.module,
|
|
runs: d.runs.length,
|
|
paths: d.paths,
|
|
lastNight: nights[d.module] ?? null,
|
|
restorePoints: mine.length,
|
|
newest: mine.length ? { snapshot: mine[mine.length - 1].short_id, at: mine[mine.length - 1].time } : null,
|
|
};
|
|
});
|
|
}
|
|
|
|
/** A module's data from a restore point, BESIDE the live data: each directory as
|
|
* <path>.restored-<stamp>. A target that already exists is refused, never overwritten. */
|
|
restore(module: string, snapshot?: string, path?: string) {
|
|
return this.serial(async () => {
|
|
const mine = await this.snapshots(module);
|
|
if (mine.length === 0) throw new Error(`${module} has no restore point on this machine`);
|
|
const chosen = snapshot ? mine.find((s) => s.id.startsWith(snapshot) || s.short_id === snapshot) : mine[mine.length - 1];
|
|
if (!chosen) {
|
|
throw new Error(`${module} has no restore point ${snapshot}; it has ${mine.map((s) => `${s.short_id} (${s.time})`).join(", ")}`);
|
|
}
|
|
const paths = path ? chosen.paths.filter((p) => p === path) : chosen.paths;
|
|
if (paths.length === 0) throw new Error(`restore point ${chosen.short_id} of ${module} holds ${chosen.paths.join(", ")}, not ${path}`);
|
|
const stamp = this.now().toISOString().replace(/[-:]/g, "").replace("T", "-").slice(0, 15);
|
|
const restored: string[] = [];
|
|
for (const p of paths) {
|
|
const target = `${p}.restored-${stamp}`;
|
|
if (existsSync(target)) throw new Error(`${target} already exists; nothing is restored over anything`);
|
|
await this.restic(["restore", `${chosen.id}:${p}`, "--target", target]);
|
|
restored.push(target);
|
|
this.say(`${module}: restored ${p} from ${chosen.short_id} to ${target}`);
|
|
}
|
|
return { module, from: { snapshot: chosen.short_id, at: chosen.time }, restored, live: "untouched — swapping it in is a person's act" };
|
|
});
|
|
}
|
|
|
|
/** The weekly look at the repository's own integrity, with a sample of the data read back. */
|
|
check(): Promise<string> {
|
|
return this.serial(() => this.restic(["check", "--read-data-subset", "5%"]));
|
|
}
|
|
}
|
|
|
|
/** When the next night is due: the given hour, local time, today if it is still ahead, else tomorrow. */
|
|
export function nextNight(now: Date, hour: number): Date {
|
|
const next = new Date(now);
|
|
next.setHours(hour, 0, 0, 0);
|
|
if (next.getTime() <= now.getTime()) next.setDate(next.getDate() + 1);
|
|
return next;
|
|
}
|
|
|
|
/** Whether a night was missed: the newest good night of any module is older than a day and a bit —
|
|
* the machine was off, or this module was not running, at the hour. */
|
|
export function missedANight(nights: Record<string, Night>, now: Date): boolean {
|
|
const good = Object.values(nights).filter((n) => n.ok).map((n) => Date.parse(n.at));
|
|
if (good.length === 0) return true;
|
|
return now.getTime() - Math.max(...good) > 26 * 3600 * 1000;
|
|
}
|