Back up every store: the restic module holds node-backup, the stores contribute their dumps
ADR 0214 / to-be 43. restic keeps one repository per machine and takes a nightly snapshot per module — 14 daily, 8 weekly, 6 monthly — and restores beside the live data, never over it. postgres, mssql and mongodb contribute a consistent dump; minio, influxdb, the vault, mailu, gitea and nextcloud the directories that hold their data.
This commit is contained in:
@@ -222,5 +222,12 @@
|
||||
"failregex": "^.*Failed authentication attempt for .* from <HOST>(?::\\d+)?\\s*$\n ^.*Invalid user .* from <HOST> port \\d+\\s*$\n ^.*User \\S+ from <HOST> not allowed because .*$",
|
||||
"jail": "backend = systemd\njournalmatch = CONTAINER_NAME=gitea\nport = http,https,222\nmaxretry = 3\nfindtime = 1d\nbantime = 1d"
|
||||
}
|
||||
],
|
||||
"contributions": [
|
||||
{
|
||||
"seat": "node-backup",
|
||||
"kind": "backup",
|
||||
"content": "path ${dir:data}\n"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -132,5 +132,12 @@
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"contributions": [
|
||||
{
|
||||
"seat": "node-backup",
|
||||
"kind": "backup",
|
||||
"content": "path ${dir:data}\npath ${dir:config}\n"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -558,5 +558,12 @@
|
||||
"failregex": "^.*(?:imap|pop3|submission|managesieve)-login: .*\\(auth failed, \\d+ attempts(?: in \\d+ secs)?\\):.*rip=<HOST>(?:,|$)",
|
||||
"jail": "backend = systemd\njournalmatch = CONTAINER_NAME=mailu-front\nport = smtp,submission,submissions,imap,imaps,pop3,pop3s\nmaxretry = 3\nfindtime = 1d\nbantime = 1d"
|
||||
}
|
||||
],
|
||||
"contributions": [
|
||||
{
|
||||
"seat": "node-backup",
|
||||
"kind": "backup",
|
||||
"content": "path ${dir:data-mail}\npath ${dir:data-dkim}\npath ${dir:data-data}\npath ${dir:data-dav}\npath ${dir:data-webmail}\n"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -79,5 +79,12 @@
|
||||
"name": "mesh-vault",
|
||||
"scope": "mesh"
|
||||
}
|
||||
],
|
||||
"contributions": [
|
||||
{
|
||||
"seat": "node-backup",
|
||||
"kind": "backup",
|
||||
"content": "path ${dir:state}\npath ${dir:ledger}\npath ${dir:root}\n"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -151,5 +151,12 @@
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"contributions": [
|
||||
{
|
||||
"seat": "node-backup",
|
||||
"kind": "backup",
|
||||
"content": "path ${dir:data}\n"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -58,6 +58,11 @@
|
||||
"type": "directory",
|
||||
"mode": "0700"
|
||||
},
|
||||
{
|
||||
"id": "dumps",
|
||||
"type": "directory",
|
||||
"mode": "0700"
|
||||
},
|
||||
{
|
||||
"id": "net",
|
||||
"type": "network",
|
||||
@@ -113,5 +118,12 @@
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"contributions": [
|
||||
{
|
||||
"seat": "node-backup",
|
||||
"kind": "backup",
|
||||
"content": "run docker exec mongodb-server sh -c 'printf \"password: %s\\n\" \"$(cat /run/secrets/root)\" > /tmp/.backup.yaml && mongodump --quiet --config /tmp/.backup.yaml --username root --authenticationDatabase admin --archive; s=$?; rm -f /tmp/.backup.yaml; exit $s' > ${dir:dumps}/all.archive.partial && mv ${dir:dumps}/all.archive.partial ${dir:dumps}/all.archive\npath ${dir:dumps}\n"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -65,6 +65,13 @@
|
||||
"mode": "0700",
|
||||
"owner": "10001:0"
|
||||
},
|
||||
{
|
||||
"id": "dumps",
|
||||
"type": "directory",
|
||||
"path": "${dir:data}/backup",
|
||||
"mode": "0700",
|
||||
"owner": "10001:0"
|
||||
},
|
||||
{
|
||||
"id": "net",
|
||||
"type": "network",
|
||||
@@ -86,6 +93,13 @@
|
||||
"${dir:data}:/var/opt/mssql"
|
||||
],
|
||||
"secrets-in-environment": "the image documents only MSSQL_SA_PASSWORD, no _FILE and no configuration field; not convertible without a wrapper entrypoint"
|
||||
},
|
||||
{
|
||||
"id": "backup-sql",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/backup.sql",
|
||||
"mode": "0600",
|
||||
"content": "SET NOCOUNT ON;\nDECLARE @n sysname, @s nvarchar(max);\nDECLARE c CURSOR LOCAL FAST_FORWARD FOR\n SELECT name FROM sys.databases WHERE database_id > 4 AND state = 0 AND source_database_id IS NULL;\nOPEN c;\nFETCH NEXT FROM c INTO @n;\nWHILE @@FETCH_STATUS = 0\nBEGIN\n SET @s = N'BACKUP DATABASE ' + QUOTENAME(@n) + N' TO DISK = N''/var/opt/mssql/backup/' + REPLACE(@n, N'''', N'''''') + N'.bak'' WITH INIT, COPY_ONLY, CHECKSUM';\n EXEC (@s);\n FETCH NEXT FROM c INTO @n;\nEND\nCLOSE c;\nDEALLOCATE c;\n"
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
@@ -112,5 +126,12 @@
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"contributions": [
|
||||
{
|
||||
"seat": "node-backup",
|
||||
"kind": "backup",
|
||||
"content": "run { cat ${dir:state}/sa.secret; echo; cat ${dir:state}/backup.sql; } | docker exec -i mssql sh -c 'read -r p; SQLCMDPASSWORD=\"$p\" exec /opt/mssql-tools18/bin/sqlcmd -C -b -S localhost -U sa -i /dev/stdin'\npath ${dir:dumps}\n"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -117,5 +117,12 @@
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"contributions": [
|
||||
{
|
||||
"seat": "node-backup",
|
||||
"kind": "backup",
|
||||
"content": "path ${dir:html}\n"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -74,6 +74,13 @@
|
||||
"mode": "0700",
|
||||
"owner": "999:70"
|
||||
},
|
||||
{
|
||||
"id": "dumps",
|
||||
"type": "directory",
|
||||
"path": "${dir:store-data}/dumps",
|
||||
"mode": "0700",
|
||||
"owner": "999:70"
|
||||
},
|
||||
{
|
||||
"id": "server",
|
||||
"type": "container",
|
||||
@@ -121,5 +128,12 @@
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"contributions": [
|
||||
{
|
||||
"seat": "node-backup",
|
||||
"kind": "backup",
|
||||
"content": "run docker exec -u postgres postgres sh -c 'cd /var/lib/postgresql/data/dumps && for db in $(psql -Atc \"select datname from pg_database where oid >= 16384 order by 1\"); do pg_dump -Fc -f \"$db.dump.partial\" \"$db\" && mv \"$db.dump.partial\" \"$db.dump\" || exit 1; done'\npath ${dir:dumps}\n"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -0,0 +1,312 @@
|
||||
// restic's own code, in the module (novox/hq ADR 0039): the machine's backups (ADR 0214, to-be 43).
|
||||
//
|
||||
// The mesh composes what to back up: every module on the machine contributes `backup` lines to the
|
||||
// node-backup seat, and the mesh writes them, each module's under a `# <module>` line and with its
|
||||
// directories already filled, into one file this module reads. Two kinds of line:
|
||||
//
|
||||
// run <shell command> run as root before the module's snapshot — a consistent dump of a store
|
||||
// path <directory> a directory the module's snapshot keeps
|
||||
//
|
||||
// Each module gets one snapshot a night, tagged with its name, so a module is listed, kept and
|
||||
// restored on its own. Everything lands in one repository on the machine — deduplicated, so every
|
||||
// night is a complete restore point and only what changed costs space — and is thinned to 14 daily,
|
||||
// 8 weekly and 6 monthly. Against mistakes, not disasters: nothing leaves the machine.
|
||||
//
|
||||
// Root's: the dumps read every store and the repository holds every module's data, and the runtime
|
||||
// loading this bundle runs as the operator's account, so restic and the run lines go through sudo
|
||||
// without a prompt where the account is not root — fail2ban's way (ADR 0175 §4).
|
||||
|
||||
import { execFile } from "node:child_process";
|
||||
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
const execFileP = promisify(execFile);
|
||||
|
||||
/** A command runner, so the backups can be tested without restic or a store. */
|
||||
export type Runner = (cmd: string, args: string[]) => Promise<string>;
|
||||
|
||||
/** The command as it is run: as given when this process is root, else through sudo without a prompt. */
|
||||
export function escalated(cmd: string, args: string[], uid: number | undefined = process.getuid?.()): [string, string[]] {
|
||||
if (uid === 0) return [cmd, args];
|
||||
return ["sudo", ["-n", cmd, ...args]];
|
||||
}
|
||||
|
||||
export const execRunner: Runner = async (cmd, args) => {
|
||||
const [program, argv] = escalated(cmd, args);
|
||||
try {
|
||||
// A night's dump of a large store takes a while; six hours is a dump that will not finish.
|
||||
const { stdout } = await execFileP(program, argv, { maxBuffer: 256 * 1024 * 1024, timeout: 6 * 3600 * 1000 });
|
||||
return stdout;
|
||||
} catch (err) {
|
||||
const e = err as { code?: string | number; stderr?: string; stdout?: string; message?: string };
|
||||
const said = `${e.stderr ?? ""}`.trim() || `${e.stdout ?? ""}`.trim();
|
||||
if (program === "sudo" && /^sudo:/m.test(said)) {
|
||||
throw new Error(`${cmd} needs root and the runtime's account may not run it without a prompt: ${said}`);
|
||||
}
|
||||
const lines = said.split("\n").map((l) => l.trim()).filter(Boolean);
|
||||
throw new Error(lines.length ? lines.slice(-3).join(" / ") : (e.message ?? `${cmd} failed`));
|
||||
}
|
||||
};
|
||||
|
||||
/** What one module declared. */
|
||||
export interface Declared {
|
||||
module: string;
|
||||
runs: string[];
|
||||
paths: string[];
|
||||
}
|
||||
|
||||
/** The composed file, read into each module's declaration, in the order the mesh wrote them. A line
|
||||
* before any module, a comment that is not a module's name, or a blank, is nothing. */
|
||||
export function parseDeclared(text: string): Declared[] {
|
||||
const out: Declared[] = [];
|
||||
let current: Declared | undefined;
|
||||
for (const raw of text.split("\n")) {
|
||||
const line = raw.trim();
|
||||
if (line === "") continue;
|
||||
const header = /^#\s*([a-z0-9][a-z0-9-]*)$/.exec(line);
|
||||
if (header) {
|
||||
current = { module: header[1], runs: [], paths: [] };
|
||||
out.push(current);
|
||||
continue;
|
||||
}
|
||||
if (line.startsWith("#") || !current) continue;
|
||||
const [kind, ...rest] = line.split(/\s+/);
|
||||
const value = line.slice(kind.length).trim();
|
||||
if (kind === "run" && value) current.runs.push(value);
|
||||
else if (kind === "path" && rest.length === 1 && value.startsWith("/")) current.paths.push(value);
|
||||
else throw new Error(`${current.module} contributes a backup line this holder does not read: ${line}`);
|
||||
}
|
||||
return out.filter((d) => d.runs.length > 0 || d.paths.length > 0);
|
||||
}
|
||||
|
||||
/** One restore point, as restic lists it. */
|
||||
export interface Snapshot {
|
||||
id: string;
|
||||
short_id: string;
|
||||
time: string;
|
||||
paths: string[];
|
||||
tags?: string[];
|
||||
hostname: string;
|
||||
}
|
||||
|
||||
/** How one module's last night went. */
|
||||
export interface Night {
|
||||
ok: boolean;
|
||||
at: string;
|
||||
snapshot?: string;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export const KEEP = { daily: 14, weekly: 8, monthly: 6 };
|
||||
|
||||
export function tagOf(module: string): string {
|
||||
return `module=${module}`;
|
||||
}
|
||||
|
||||
export interface Where {
|
||||
declared: string;
|
||||
repository: string;
|
||||
passwordFile: string;
|
||||
state: string;
|
||||
}
|
||||
|
||||
export function whereFromEnv(env: NodeJS.ProcessEnv = process.env): Where {
|
||||
const need = (k: string) => {
|
||||
const v = env[k];
|
||||
if (!v) throw new Error(`${k} is not set; the mesh gives it to this module's tools`);
|
||||
return v;
|
||||
};
|
||||
return {
|
||||
declared: need("MESH_BACKUP_DECLARED"),
|
||||
repository: need("MESH_BACKUP_REPOSITORY"),
|
||||
passwordFile: need("MESH_BACKUP_PASSWORD_FILE"),
|
||||
state: need("MESH_BACKUP_STATE"),
|
||||
};
|
||||
}
|
||||
|
||||
export class Backups {
|
||||
private busy: Promise<unknown> = Promise.resolve();
|
||||
readonly where: Where;
|
||||
private run: Runner;
|
||||
private readonly now: () => Date;
|
||||
private readonly say: (line: string) => void;
|
||||
|
||||
constructor(
|
||||
where: Where,
|
||||
run: Runner = execRunner,
|
||||
now: () => Date = () => new Date(),
|
||||
say: (line: string) => void = (l) => console.error(`[restic] ${l}`),
|
||||
) {
|
||||
this.where = where;
|
||||
this.run = run;
|
||||
this.now = now;
|
||||
this.say = say;
|
||||
}
|
||||
|
||||
private restic(args: string[]): Promise<string> {
|
||||
return this.run("restic", ["--repo", this.where.repository, "--password-file", this.where.passwordFile, "--no-cache", ...args]);
|
||||
}
|
||||
|
||||
/** One thing at a time: two nights, or a night and a restore, never share a dump. */
|
||||
private serial<T>(work: () => Promise<T>): Promise<T> {
|
||||
const next = this.busy.then(work, work);
|
||||
this.busy = next.catch(() => undefined);
|
||||
return next;
|
||||
}
|
||||
|
||||
declared(): Declared[] {
|
||||
return parseDeclared(readFileSync(this.where.declared, "utf8"));
|
||||
}
|
||||
|
||||
private nightsFile(): string {
|
||||
return join(this.where.state, "nights.json");
|
||||
}
|
||||
|
||||
nights(): Record<string, Night> {
|
||||
try {
|
||||
return JSON.parse(readFileSync(this.nightsFile(), "utf8")) as Record<string, Night>;
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
private record(module: string, night: Night): void {
|
||||
const all = this.nights();
|
||||
all[module] = night;
|
||||
writeFileSync(this.nightsFile(), JSON.stringify(all, null, 2) + "\n", { mode: 0o600 });
|
||||
}
|
||||
|
||||
/** The repository, made the first time. A repository that exists and cannot be opened is said,
|
||||
* never replaced: replacing it would discard every restore point to fix a password. */
|
||||
async ensureRepository(): Promise<void> {
|
||||
try {
|
||||
await this.restic(["cat", "config"]);
|
||||
} catch (err) {
|
||||
const why = String((err as Error).message);
|
||||
if (!/does not exist|unable to open config file|Is there a repository at the following location/i.test(why)) {
|
||||
throw new Error(`the repository at ${this.where.repository} cannot be opened, and is left as it is: ${why}`);
|
||||
}
|
||||
this.say(`no repository at ${this.where.repository}; making one`);
|
||||
await this.restic(["init"]);
|
||||
}
|
||||
}
|
||||
|
||||
/** One module's night: its run lines, then one snapshot of its paths. A failure is that module's. */
|
||||
private async one(d: Declared): Promise<Night> {
|
||||
const at = this.now().toISOString();
|
||||
try {
|
||||
for (const command of d.runs) {
|
||||
await this.run("sh", ["-c", command]);
|
||||
}
|
||||
const missing = d.paths.filter((p) => !existsSync(p));
|
||||
if (missing.length > 0) throw new Error(`${missing.join(", ")} does not exist`);
|
||||
if (d.paths.length === 0) throw new Error("it runs a dump and names no directory to keep it from");
|
||||
const out = await this.restic(["backup", "--json", "--tag", tagOf(d.module), ...d.paths]);
|
||||
const summary = out
|
||||
.split("\n")
|
||||
.map((l) => { try { return JSON.parse(l) as { message_type?: string; snapshot_id?: string }; } catch { return {}; } })
|
||||
.find((m) => m.message_type === "summary");
|
||||
const night: Night = { ok: true, at, snapshot: summary?.snapshot_id?.slice(0, 8) };
|
||||
this.say(`${d.module}: backed up (${night.snapshot ?? "no snapshot id reported"})`);
|
||||
return night;
|
||||
} catch (err) {
|
||||
const night: Night = { ok: false, at, error: String((err as Error).message) };
|
||||
this.say(`${d.module}: NOT backed up: ${night.error}`);
|
||||
return night;
|
||||
}
|
||||
}
|
||||
|
||||
/** A night: every module, or one, then the rotation. Returns each module's outcome. */
|
||||
backUp(only?: string): Promise<Record<string, Night>> {
|
||||
return this.serial(async () => {
|
||||
await this.ensureRepository();
|
||||
const all = this.declared();
|
||||
const chosen = only ? all.filter((d) => d.module === only) : all;
|
||||
if (only && chosen.length === 0) {
|
||||
throw new Error(`${only} declares nothing to back up on this machine; it backs up ${all.map((d) => d.module).join(", ") || "nothing"}`);
|
||||
}
|
||||
const outcome: Record<string, Night> = {};
|
||||
for (const d of chosen) {
|
||||
outcome[d.module] = await this.one(d);
|
||||
this.record(d.module, outcome[d.module]);
|
||||
}
|
||||
await this.restic([
|
||||
"forget", "--prune", "--group-by", "host,tags",
|
||||
"--keep-daily", String(KEEP.daily), "--keep-weekly", String(KEEP.weekly), "--keep-monthly", String(KEEP.monthly),
|
||||
]).catch((err) => this.say(`thinning the restore points failed, and every one is kept: ${(err as Error).message}`));
|
||||
return outcome;
|
||||
});
|
||||
}
|
||||
|
||||
async snapshots(module?: string): Promise<Snapshot[]> {
|
||||
const args = ["snapshots", "--json"];
|
||||
if (module) args.push("--tag", tagOf(module));
|
||||
return JSON.parse((await this.restic(args)) || "[]") as Snapshot[];
|
||||
}
|
||||
|
||||
/** What is backed up here: each module, what it declared, its last night and its restore points. */
|
||||
async backedUp(module?: string) {
|
||||
const nights = this.nights();
|
||||
const snaps = await this.snapshots(module).catch(() => [] as Snapshot[]);
|
||||
return this.declared()
|
||||
.filter((d) => !module || d.module === module)
|
||||
.map((d) => {
|
||||
const mine = snaps.filter((s) => (s.tags ?? []).includes(tagOf(d.module)));
|
||||
return {
|
||||
module: d.module,
|
||||
runs: d.runs.length,
|
||||
paths: d.paths,
|
||||
lastNight: nights[d.module] ?? null,
|
||||
restorePoints: mine.length,
|
||||
newest: mine.length ? { snapshot: mine[mine.length - 1].short_id, at: mine[mine.length - 1].time } : null,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/** A module's data from a restore point, BESIDE the live data: each directory as
|
||||
* <path>.restored-<stamp>. A target that already exists is refused, never overwritten. */
|
||||
restore(module: string, snapshot?: string, path?: string) {
|
||||
return this.serial(async () => {
|
||||
const mine = await this.snapshots(module);
|
||||
if (mine.length === 0) throw new Error(`${module} has no restore point on this machine`);
|
||||
const chosen = snapshot ? mine.find((s) => s.id.startsWith(snapshot) || s.short_id === snapshot) : mine[mine.length - 1];
|
||||
if (!chosen) {
|
||||
throw new Error(`${module} has no restore point ${snapshot}; it has ${mine.map((s) => `${s.short_id} (${s.time})`).join(", ")}`);
|
||||
}
|
||||
const paths = path ? chosen.paths.filter((p) => p === path) : chosen.paths;
|
||||
if (paths.length === 0) throw new Error(`restore point ${chosen.short_id} of ${module} holds ${chosen.paths.join(", ")}, not ${path}`);
|
||||
const stamp = this.now().toISOString().replace(/[-:]/g, "").replace("T", "-").slice(0, 15);
|
||||
const restored: string[] = [];
|
||||
for (const p of paths) {
|
||||
const target = `${p}.restored-${stamp}`;
|
||||
if (existsSync(target)) throw new Error(`${target} already exists; nothing is restored over anything`);
|
||||
await this.restic(["restore", `${chosen.id}:${p}`, "--target", target]);
|
||||
restored.push(target);
|
||||
this.say(`${module}: restored ${p} from ${chosen.short_id} to ${target}`);
|
||||
}
|
||||
return { module, from: { snapshot: chosen.short_id, at: chosen.time }, restored, live: "untouched — swapping it in is a person's act" };
|
||||
});
|
||||
}
|
||||
|
||||
/** The weekly look at the repository's own integrity, with a sample of the data read back. */
|
||||
check(): Promise<string> {
|
||||
return this.serial(() => this.restic(["check", "--read-data-subset", "5%"]));
|
||||
}
|
||||
}
|
||||
|
||||
/** When the next night is due: the given hour, local time, today if it is still ahead, else tomorrow. */
|
||||
export function nextNight(now: Date, hour: number): Date {
|
||||
const next = new Date(now);
|
||||
next.setHours(hour, 0, 0, 0);
|
||||
if (next.getTime() <= now.getTime()) next.setDate(next.getDate() + 1);
|
||||
return next;
|
||||
}
|
||||
|
||||
/** Whether a night was missed: the newest good night of any module is older than a day and a bit —
|
||||
* the machine was off, or this module was not running, at the hour. */
|
||||
export function missedANight(nights: Record<string, Night>, now: Date): boolean {
|
||||
const good = Object.values(nights).filter((n) => n.ok).map((n) => Date.parse(n.at));
|
||||
if (good.length === 0) return true;
|
||||
return now.getTime() - Math.max(...good) > 26 * 3600 * 1000;
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
{
|
||||
"module": "restic",
|
||||
"version": "1",
|
||||
"claims": [
|
||||
{
|
||||
"name": "node-backup",
|
||||
"scope": "node",
|
||||
"serves": [
|
||||
"backed-up",
|
||||
"now",
|
||||
"restore"
|
||||
]
|
||||
}
|
||||
],
|
||||
"own-secrets": {
|
||||
"repository": "${dir:state}/repository.secret"
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"id": "state",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"place": "."
|
||||
},
|
||||
{
|
||||
"id": "repository",
|
||||
"type": "directory",
|
||||
"mode": "0700"
|
||||
},
|
||||
{
|
||||
"id": "declared",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/backups.conf",
|
||||
"mode": "0600",
|
||||
"content": "# What the modules on this machine back up, composed by the mesh (novox/hq to-be 43). Do not edit.\n${contribution:node-backup:backup}"
|
||||
},
|
||||
{
|
||||
"id": "tool",
|
||||
"type": "package",
|
||||
"package": "restic"
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "tools",
|
||||
"kind": "bundle",
|
||||
"language": "typescript",
|
||||
"entrypoints": [
|
||||
"tools/index.js"
|
||||
],
|
||||
"loads": [
|
||||
"tools/index.js"
|
||||
],
|
||||
"env": {
|
||||
"MESH_BACKUP_DECLARED": "${dir:state}/backups.conf",
|
||||
"MESH_BACKUP_REPOSITORY": "${dir:repository}",
|
||||
"MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret",
|
||||
"MESH_BACKUP_STATE": "${dir:state}"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"name": "@novox/module-restic",
|
||||
"version": "0.1.0",
|
||||
"description": "restic \u2014 the machine's backups: holds the node-backup seat, takes a nightly restore point of what every module on the machine declares, keeps 14 daily, 8 weekly and 6 monthly, and restores beside the live data (novox/hq ADR 0214, to-be 43).",
|
||||
"type": "module",
|
||||
"private": true,
|
||||
"dependencies": {
|
||||
"@novox/mesh-sdk": "^0.1.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.0.0",
|
||||
"typescript": "^5.6.0"
|
||||
},
|
||||
"scripts": {
|
||||
"build": "tsc client.ts tools/index.ts --module NodeNext --moduleResolution NodeNext --target ES2022 --rootDir . --outDir dist",
|
||||
"test": "node --test --experimental-strip-types 'test/*.test.ts'"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
// The machine's backups over a fake restic and fake stores (novox/hq ADR 0214, to-be 43), and — where
|
||||
// restic is installed — over the real one, on throwaway directories.
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { Backups, escalated, missedANight, nextNight, parseDeclared, type Runner } from "../client.ts";
|
||||
|
||||
const COMPOSED =
|
||||
"# What the modules on this machine back up, composed by the mesh. Do not edit.\n" +
|
||||
"# postgres\nrun docker exec -u postgres postgres sh -c 'pg-dump-all'\npath /var/lib/mesh-store/dumps\n" +
|
||||
"# mailu\npath /var/lib/mailu/data-mail\npath /var/lib/mailu/data-dkim\n";
|
||||
|
||||
function place(declared: string) {
|
||||
const dir = mkdtempSync(join(tmpdir(), "restic-test-"));
|
||||
writeFileSync(join(dir, "backups.conf"), declared);
|
||||
writeFileSync(join(dir, "pw"), "secret\n");
|
||||
return { declared: join(dir, "backups.conf"), repository: join(dir, "repo"), passwordFile: join(dir, "pw"), state: dir };
|
||||
}
|
||||
|
||||
test("the composed file is read into each module's runs and paths, the header comment being nothing", () => {
|
||||
assert.deepEqual(parseDeclared(COMPOSED), [
|
||||
{ module: "postgres", runs: ["docker exec -u postgres postgres sh -c 'pg-dump-all'"], paths: ["/var/lib/mesh-store/dumps"] },
|
||||
{ module: "mailu", runs: [], paths: ["/var/lib/mailu/data-mail", "/var/lib/mailu/data-dkim"] },
|
||||
]);
|
||||
});
|
||||
|
||||
test("a line this holder does not read is refused, naming the module, rather than skipped", () => {
|
||||
assert.throws(() => parseDeclared("# pg\ncopy /x\n"), /pg contributes a backup line this holder does not read: copy \/x/);
|
||||
assert.throws(() => parseDeclared("# pg\npath relative/dir\n"), /pg contributes/);
|
||||
});
|
||||
|
||||
test("restic and the dumps run through sudo where the account is not root", () => {
|
||||
assert.deepEqual(escalated("restic", ["snapshots"], 1000), ["sudo", ["-n", "restic", "snapshots"]]);
|
||||
assert.deepEqual(escalated("restic", ["snapshots"], 0), ["restic", ["snapshots"]]);
|
||||
});
|
||||
|
||||
test("a night runs each module's dump before its snapshot, and one failing module fails only itself", async () => {
|
||||
const where = place("# pg\nrun dump-it\npath /\n# broken\nrun fail-it\npath /\n# mail\npath /\n");
|
||||
const calls: string[] = [];
|
||||
const run: Runner = async (cmd, args) => {
|
||||
const line = cmd === "restic" ? `restic ${args.slice(5).join(" ")}` : `${cmd} ${args.join(" ")}`;
|
||||
calls.push(line);
|
||||
if (line === "sh -c fail-it") throw new Error("the dump failed");
|
||||
if (line.startsWith("restic backup")) return '{"message_type":"status"}\n{"message_type":"summary","snapshot_id":"abcdef0123456789"}\n';
|
||||
return "";
|
||||
};
|
||||
const outcome = await new Backups(where, run, () => new Date("2026-10-06T03:00:00Z"), () => {}).backUp();
|
||||
assert.equal(outcome.pg.ok, true);
|
||||
assert.equal(outcome.pg.snapshot, "abcdef01");
|
||||
assert.equal(outcome.broken.ok, false);
|
||||
assert.match(outcome.broken.error ?? "", /the dump failed/);
|
||||
assert.equal(outcome.mail.ok, true);
|
||||
assert.deepEqual(calls, [
|
||||
"restic cat config",
|
||||
"sh -c dump-it",
|
||||
"restic backup --json --tag module=pg /",
|
||||
"sh -c fail-it",
|
||||
"restic backup --json --tag module=mail /",
|
||||
"restic forget --prune --group-by host,tags --keep-daily 14 --keep-weekly 8 --keep-monthly 6",
|
||||
]);
|
||||
// Recorded, so `backed-up` and the missed-night check read it.
|
||||
const nights = JSON.parse(readFileSync(join(where.state, "nights.json"), "utf8"));
|
||||
assert.equal(nights.broken.ok, false);
|
||||
assert.equal(nights.mail.ok, true);
|
||||
});
|
||||
|
||||
test("a repository that exists and will not open is never replaced", async () => {
|
||||
const where = place("# pg\npath /\n");
|
||||
const calls: string[] = [];
|
||||
const run: Runner = async (cmd, args) => {
|
||||
calls.push(args.slice(5).join(" "));
|
||||
if (args.includes("cat")) throw new Error("Fatal: wrong password or no key found");
|
||||
return "";
|
||||
};
|
||||
await assert.rejects(new Backups(where, run, undefined, () => {}).backUp(), /cannot be opened, and is left as it is/);
|
||||
assert.ok(!calls.includes("init"), "it made a new repository over one it could not open");
|
||||
});
|
||||
|
||||
test("a declared directory that does not exist fails that module's night", async () => {
|
||||
const where = place("# pg\npath /nowhere/at/all\n");
|
||||
const run: Runner = async () => "";
|
||||
const outcome = await new Backups(where, run, undefined, () => {}).backUp();
|
||||
assert.equal(outcome.pg.ok, false);
|
||||
assert.match(outcome.pg.error ?? "", /\/nowhere\/at\/all does not exist/);
|
||||
});
|
||||
|
||||
test("a night is due at the hour today while it is ahead, else tomorrow; a missed one is noticed", () => {
|
||||
const morning = new Date(2026, 9, 6, 1, 30);
|
||||
assert.equal(nextNight(morning, 3).getTime(), new Date(2026, 9, 6, 3, 0).getTime());
|
||||
const afternoon = new Date(2026, 9, 6, 15, 0);
|
||||
assert.equal(nextNight(afternoon, 3).getTime(), new Date(2026, 9, 7, 3, 0).getTime());
|
||||
assert.equal(missedANight({}, afternoon), true);
|
||||
assert.equal(missedANight({ pg: { ok: true, at: new Date(2026, 9, 6, 3, 5).toISOString() } }, afternoon), false);
|
||||
assert.equal(missedANight({ pg: { ok: true, at: new Date(2026, 9, 4, 3, 5).toISOString() } }, afternoon), true);
|
||||
assert.equal(missedANight({ pg: { ok: false, at: new Date(2026, 9, 6, 3, 5).toISOString() } }, afternoon), true);
|
||||
});
|
||||
|
||||
// The real thing, where restic is installed: a dump, a snapshot, a mistake, and a restore beside.
|
||||
const hasRestic = (() => {
|
||||
try {
|
||||
execFileSync("restic", ["version"], { stdio: "ignore" });
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
})();
|
||||
|
||||
test("with the real restic: a mistake is undone by a restore beside the live data", { skip: !hasRestic && "restic is not installed" }, async () => {
|
||||
const root = mkdtempSync(join(tmpdir(), "restic-real-"));
|
||||
const store = join(root, "store");
|
||||
const dumps = join(root, "dumps");
|
||||
mkdirSync(store);
|
||||
mkdirSync(dumps);
|
||||
writeFileSync(join(store, "mailbox"), "the only copy of a letter\n");
|
||||
const where = place(`# mail\npath ${store}\n# pg\nrun echo 'every row' > ${dumps}/all.dump\npath ${dumps}\n`);
|
||||
const backups = new Backups(where, undefined, () => new Date("2026-10-06T03:00:00Z"), () => {});
|
||||
// escalated() would sudo; the test runs as whoever it is, against its own repository.
|
||||
(backups as unknown as { run: Runner }).run = async (cmd, args) => execFileSync(cmd, args, { encoding: "utf8" });
|
||||
|
||||
const night = await backups.backUp();
|
||||
assert.equal(night.mail.ok, true, night.mail.error);
|
||||
assert.equal(night.pg.ok, true, night.pg.error);
|
||||
assert.equal(readFileSync(join(dumps, "all.dump"), "utf8"), "every row\n");
|
||||
|
||||
// The mistake.
|
||||
rmSync(join(store, "mailbox"));
|
||||
|
||||
const listed = await backups.backedUp();
|
||||
assert.deepEqual(listed.map((m) => [m.module, m.restorePoints]), [["mail", 1], ["pg", 1]]);
|
||||
|
||||
const restored = await backups.restore("mail");
|
||||
assert.equal(restored.restored.length, 1);
|
||||
assert.match(restored.restored[0], /store\.restored-20261006-030000$/);
|
||||
assert.equal(readFileSync(join(restored.restored[0], "mailbox"), "utf8"), "the only copy of a letter\n");
|
||||
assert.ok(!existsSync(join(store, "mailbox")), "the restore wrote into the live directory");
|
||||
|
||||
// Never over anything: the same restore again finds its target taken.
|
||||
await assert.rejects(backups.restore("mail"), /already exists; nothing is restored over anything/);
|
||||
});
|
||||
@@ -0,0 +1,78 @@
|
||||
// The machine's backups: the node-backup seat's three verbs — what is backed up, take one now,
|
||||
// restore beside the live data — and the night that runs without anyone asking (novox/hq ADR 0214,
|
||||
// to-be 43). What is backed up is composed by the mesh from the modules the machine runs; this code
|
||||
// only runs it.
|
||||
|
||||
import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools";
|
||||
import { Backups, missedANight, nextNight, whereFromEnv } from "../client.js";
|
||||
|
||||
export function getSeatVerbs(backups: Backups): ToolDefinition[] {
|
||||
return [
|
||||
{
|
||||
name: "backed-up",
|
||||
description:
|
||||
"What this machine backs up: each module, what it declared, its last good night, how many restore points are kept.",
|
||||
input: { module: { type: "string", description: "one module (optional)" } },
|
||||
run: async (args) => backups.backedUp(args.module ? String(args.module) : undefined),
|
||||
},
|
||||
{
|
||||
name: "now",
|
||||
description:
|
||||
"Take a backup now, of one module or of every module on this machine — before a migration, a retirement or anything else that could go wrong. Answers when it has started; `backed-up` says how it went.",
|
||||
input: { module: { type: "string", description: "one module (optional)" } },
|
||||
run: async (args) => {
|
||||
const only = args.module ? String(args.module) : undefined;
|
||||
// A night of a large store outlasts any call; it is started, and its outcome recorded.
|
||||
backups.backUp(only).catch((err) => console.error(`[restic] a backup asked for now failed: ${(err as Error).message}`));
|
||||
return { started: only ?? "every module on this machine", follow: "backed-up" };
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "restore",
|
||||
description:
|
||||
"Restore one module's data from a restore point BESIDE the live data, never over it: each directory as <path>.restored-<date>. Swapping it in is a person's act.",
|
||||
input: {
|
||||
module: { type: "string", description: "the module" },
|
||||
snapshot: { type: "string", description: "the restore point (the newest when omitted)" },
|
||||
path: { type: "string", description: "one of the module's directories (all of them when omitted)" },
|
||||
},
|
||||
run: async (args) =>
|
||||
backups.restore(String(args.module ?? ""), args.snapshot ? String(args.snapshot) : undefined, args.path ? String(args.path) : undefined),
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
const backups = new Backups(whereFromEnv());
|
||||
registerModuleTools("node-backup", () => getSeatVerbs(backups));
|
||||
|
||||
// The night. At the hour, every module; and at start, if a night was missed — the machine was off
|
||||
// or this module was not running at the hour — one now rather than a day later.
|
||||
const HOUR = Number(process.env.MESH_BACKUP_HOUR ?? "3");
|
||||
|
||||
async function night(): Promise<void> {
|
||||
try {
|
||||
const outcome = await backups.backUp();
|
||||
const failed = Object.entries(outcome).filter(([, n]) => !n.ok).map(([m]) => m);
|
||||
if (failed.length > 0) console.error(`[restic] the night left ${failed.join(", ")} without a backup`);
|
||||
// Sundays, the repository's own integrity with a sample of the data read back.
|
||||
if (new Date().getDay() === 0) {
|
||||
await backups.check().then(
|
||||
() => console.error("[restic] the repository checks out"),
|
||||
(err) => console.error(`[restic] the repository does NOT check out: ${(err as Error).message}`),
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`[restic] the night did not run: ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
function schedule(): void {
|
||||
const at = nextNight(new Date(), HOUR);
|
||||
setTimeout(() => void night().finally(schedule), at.getTime() - Date.now());
|
||||
}
|
||||
|
||||
if (missedANight(backups.nights(), new Date())) {
|
||||
// Not at once: a machine just started has its stores still coming up.
|
||||
setTimeout(() => void night(), 10 * 60 * 1000);
|
||||
}
|
||||
schedule();
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"noEmit": true
|
||||
},
|
||||
"include": [
|
||||
"client.ts",
|
||||
"tools/index.ts"
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user