ADR 0214 / to-be 43. restic keeps one repository per machine and takes a nightly snapshot per module — 14 daily, 8 weekly, 6 monthly — and restores beside the live data, never over it. postgres, mssql and mongodb contribute a consistent dump; minio, influxdb, the vault, mailu, gitea and nextcloud the directories that hold their data.
143 lines
7.3 KiB
TypeScript
143 lines
7.3 KiB
TypeScript
// The machine's backups over a fake restic and fake stores (novox/hq ADR 0214, to-be 43), and — where
|
|
// restic is installed — over the real one, on throwaway directories.
|
|
import { test } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { execFileSync } from "node:child_process";
|
|
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import { Backups, escalated, missedANight, nextNight, parseDeclared, type Runner } from "../client.ts";
|
|
|
|
const COMPOSED =
|
|
"# What the modules on this machine back up, composed by the mesh. Do not edit.\n" +
|
|
"# postgres\nrun docker exec -u postgres postgres sh -c 'pg-dump-all'\npath /var/lib/mesh-store/dumps\n" +
|
|
"# mailu\npath /var/lib/mailu/data-mail\npath /var/lib/mailu/data-dkim\n";
|
|
|
|
function place(declared: string) {
|
|
const dir = mkdtempSync(join(tmpdir(), "restic-test-"));
|
|
writeFileSync(join(dir, "backups.conf"), declared);
|
|
writeFileSync(join(dir, "pw"), "secret\n");
|
|
return { declared: join(dir, "backups.conf"), repository: join(dir, "repo"), passwordFile: join(dir, "pw"), state: dir };
|
|
}
|
|
|
|
test("the composed file is read into each module's runs and paths, the header comment being nothing", () => {
|
|
assert.deepEqual(parseDeclared(COMPOSED), [
|
|
{ module: "postgres", runs: ["docker exec -u postgres postgres sh -c 'pg-dump-all'"], paths: ["/var/lib/mesh-store/dumps"] },
|
|
{ module: "mailu", runs: [], paths: ["/var/lib/mailu/data-mail", "/var/lib/mailu/data-dkim"] },
|
|
]);
|
|
});
|
|
|
|
test("a line this holder does not read is refused, naming the module, rather than skipped", () => {
|
|
assert.throws(() => parseDeclared("# pg\ncopy /x\n"), /pg contributes a backup line this holder does not read: copy \/x/);
|
|
assert.throws(() => parseDeclared("# pg\npath relative/dir\n"), /pg contributes/);
|
|
});
|
|
|
|
test("restic and the dumps run through sudo where the account is not root", () => {
|
|
assert.deepEqual(escalated("restic", ["snapshots"], 1000), ["sudo", ["-n", "restic", "snapshots"]]);
|
|
assert.deepEqual(escalated("restic", ["snapshots"], 0), ["restic", ["snapshots"]]);
|
|
});
|
|
|
|
test("a night runs each module's dump before its snapshot, and one failing module fails only itself", async () => {
|
|
const where = place("# pg\nrun dump-it\npath /\n# broken\nrun fail-it\npath /\n# mail\npath /\n");
|
|
const calls: string[] = [];
|
|
const run: Runner = async (cmd, args) => {
|
|
const line = cmd === "restic" ? `restic ${args.slice(5).join(" ")}` : `${cmd} ${args.join(" ")}`;
|
|
calls.push(line);
|
|
if (line === "sh -c fail-it") throw new Error("the dump failed");
|
|
if (line.startsWith("restic backup")) return '{"message_type":"status"}\n{"message_type":"summary","snapshot_id":"abcdef0123456789"}\n';
|
|
return "";
|
|
};
|
|
const outcome = await new Backups(where, run, () => new Date("2026-10-06T03:00:00Z"), () => {}).backUp();
|
|
assert.equal(outcome.pg.ok, true);
|
|
assert.equal(outcome.pg.snapshot, "abcdef01");
|
|
assert.equal(outcome.broken.ok, false);
|
|
assert.match(outcome.broken.error ?? "", /the dump failed/);
|
|
assert.equal(outcome.mail.ok, true);
|
|
assert.deepEqual(calls, [
|
|
"restic cat config",
|
|
"sh -c dump-it",
|
|
"restic backup --json --tag module=pg /",
|
|
"sh -c fail-it",
|
|
"restic backup --json --tag module=mail /",
|
|
"restic forget --prune --group-by host,tags --keep-daily 14 --keep-weekly 8 --keep-monthly 6",
|
|
]);
|
|
// Recorded, so `backed-up` and the missed-night check read it.
|
|
const nights = JSON.parse(readFileSync(join(where.state, "nights.json"), "utf8"));
|
|
assert.equal(nights.broken.ok, false);
|
|
assert.equal(nights.mail.ok, true);
|
|
});
|
|
|
|
test("a repository that exists and will not open is never replaced", async () => {
|
|
const where = place("# pg\npath /\n");
|
|
const calls: string[] = [];
|
|
const run: Runner = async (cmd, args) => {
|
|
calls.push(args.slice(5).join(" "));
|
|
if (args.includes("cat")) throw new Error("Fatal: wrong password or no key found");
|
|
return "";
|
|
};
|
|
await assert.rejects(new Backups(where, run, undefined, () => {}).backUp(), /cannot be opened, and is left as it is/);
|
|
assert.ok(!calls.includes("init"), "it made a new repository over one it could not open");
|
|
});
|
|
|
|
test("a declared directory that does not exist fails that module's night", async () => {
|
|
const where = place("# pg\npath /nowhere/at/all\n");
|
|
const run: Runner = async () => "";
|
|
const outcome = await new Backups(where, run, undefined, () => {}).backUp();
|
|
assert.equal(outcome.pg.ok, false);
|
|
assert.match(outcome.pg.error ?? "", /\/nowhere\/at\/all does not exist/);
|
|
});
|
|
|
|
test("a night is due at the hour today while it is ahead, else tomorrow; a missed one is noticed", () => {
|
|
const morning = new Date(2026, 9, 6, 1, 30);
|
|
assert.equal(nextNight(morning, 3).getTime(), new Date(2026, 9, 6, 3, 0).getTime());
|
|
const afternoon = new Date(2026, 9, 6, 15, 0);
|
|
assert.equal(nextNight(afternoon, 3).getTime(), new Date(2026, 9, 7, 3, 0).getTime());
|
|
assert.equal(missedANight({}, afternoon), true);
|
|
assert.equal(missedANight({ pg: { ok: true, at: new Date(2026, 9, 6, 3, 5).toISOString() } }, afternoon), false);
|
|
assert.equal(missedANight({ pg: { ok: true, at: new Date(2026, 9, 4, 3, 5).toISOString() } }, afternoon), true);
|
|
assert.equal(missedANight({ pg: { ok: false, at: new Date(2026, 9, 6, 3, 5).toISOString() } }, afternoon), true);
|
|
});
|
|
|
|
// The real thing, where restic is installed: a dump, a snapshot, a mistake, and a restore beside.
|
|
const hasRestic = (() => {
|
|
try {
|
|
execFileSync("restic", ["version"], { stdio: "ignore" });
|
|
return true;
|
|
} catch {
|
|
return false;
|
|
}
|
|
})();
|
|
|
|
test("with the real restic: a mistake is undone by a restore beside the live data", { skip: !hasRestic && "restic is not installed" }, async () => {
|
|
const root = mkdtempSync(join(tmpdir(), "restic-real-"));
|
|
const store = join(root, "store");
|
|
const dumps = join(root, "dumps");
|
|
mkdirSync(store);
|
|
mkdirSync(dumps);
|
|
writeFileSync(join(store, "mailbox"), "the only copy of a letter\n");
|
|
const where = place(`# mail\npath ${store}\n# pg\nrun echo 'every row' > ${dumps}/all.dump\npath ${dumps}\n`);
|
|
const backups = new Backups(where, undefined, () => new Date("2026-10-06T03:00:00Z"), () => {});
|
|
// escalated() would sudo; the test runs as whoever it is, against its own repository.
|
|
(backups as unknown as { run: Runner }).run = async (cmd, args) => execFileSync(cmd, args, { encoding: "utf8" });
|
|
|
|
const night = await backups.backUp();
|
|
assert.equal(night.mail.ok, true, night.mail.error);
|
|
assert.equal(night.pg.ok, true, night.pg.error);
|
|
assert.equal(readFileSync(join(dumps, "all.dump"), "utf8"), "every row\n");
|
|
|
|
// The mistake.
|
|
rmSync(join(store, "mailbox"));
|
|
|
|
const listed = await backups.backedUp();
|
|
assert.deepEqual(listed.map((m) => [m.module, m.restorePoints]), [["mail", 1], ["pg", 1]]);
|
|
|
|
const restored = await backups.restore("mail");
|
|
assert.equal(restored.restored.length, 1);
|
|
assert.match(restored.restored[0], /store\.restored-20261006-030000$/);
|
|
assert.equal(readFileSync(join(restored.restored[0], "mailbox"), "utf8"), "the only copy of a letter\n");
|
|
assert.ok(!existsSync(join(store, "mailbox")), "the restore wrote into the live directory");
|
|
|
|
// Never over anything: the same restore again finds its target taken.
|
|
await assert.rejects(backups.restore("mail"), /already exists; nothing is restored over anything/);
|
|
});
|