Files
mesh-catalog/modules/restic/test/client.test.ts
T
jschoubben 32cd92baeb Back up every store: the restic module holds node-backup, the stores contribute their dumps
ADR 0214 / to-be 43. restic keeps one repository per machine and takes a nightly snapshot per
module — 14 daily, 8 weekly, 6 monthly — and restores beside the live data, never over it. postgres,
mssql and mongodb contribute a consistent dump; minio, influxdb, the vault, mailu, gitea and
nextcloud the directories that hold their data.
2026-10-05 11:47:59 +02:00

143 lines
7.3 KiB
TypeScript

// The machine's backups over a fake restic and fake stores (novox/hq ADR 0214, to-be 43), and — where
// restic is installed — over the real one, on throwaway directories.
import { test } from "node:test";
import assert from "node:assert/strict";
import { execFileSync } from "node:child_process";
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { Backups, escalated, missedANight, nextNight, parseDeclared, type Runner } from "../client.ts";
const COMPOSED =
"# What the modules on this machine back up, composed by the mesh. Do not edit.\n" +
"# postgres\nrun docker exec -u postgres postgres sh -c 'pg-dump-all'\npath /var/lib/mesh-store/dumps\n" +
"# mailu\npath /var/lib/mailu/data-mail\npath /var/lib/mailu/data-dkim\n";
function place(declared: string) {
const dir = mkdtempSync(join(tmpdir(), "restic-test-"));
writeFileSync(join(dir, "backups.conf"), declared);
writeFileSync(join(dir, "pw"), "secret\n");
return { declared: join(dir, "backups.conf"), repository: join(dir, "repo"), passwordFile: join(dir, "pw"), state: dir };
}
test("the composed file is read into each module's runs and paths, the header comment being nothing", () => {
assert.deepEqual(parseDeclared(COMPOSED), [
{ module: "postgres", runs: ["docker exec -u postgres postgres sh -c 'pg-dump-all'"], paths: ["/var/lib/mesh-store/dumps"] },
{ module: "mailu", runs: [], paths: ["/var/lib/mailu/data-mail", "/var/lib/mailu/data-dkim"] },
]);
});
test("a line this holder does not read is refused, naming the module, rather than skipped", () => {
assert.throws(() => parseDeclared("# pg\ncopy /x\n"), /pg contributes a backup line this holder does not read: copy \/x/);
assert.throws(() => parseDeclared("# pg\npath relative/dir\n"), /pg contributes/);
});
test("restic and the dumps run through sudo where the account is not root", () => {
assert.deepEqual(escalated("restic", ["snapshots"], 1000), ["sudo", ["-n", "restic", "snapshots"]]);
assert.deepEqual(escalated("restic", ["snapshots"], 0), ["restic", ["snapshots"]]);
});
test("a night runs each module's dump before its snapshot, and one failing module fails only itself", async () => {
const where = place("# pg\nrun dump-it\npath /\n# broken\nrun fail-it\npath /\n# mail\npath /\n");
const calls: string[] = [];
const run: Runner = async (cmd, args) => {
const line = cmd === "restic" ? `restic ${args.slice(5).join(" ")}` : `${cmd} ${args.join(" ")}`;
calls.push(line);
if (line === "sh -c fail-it") throw new Error("the dump failed");
if (line.startsWith("restic backup")) return '{"message_type":"status"}\n{"message_type":"summary","snapshot_id":"abcdef0123456789"}\n';
return "";
};
const outcome = await new Backups(where, run, () => new Date("2026-10-06T03:00:00Z"), () => {}).backUp();
assert.equal(outcome.pg.ok, true);
assert.equal(outcome.pg.snapshot, "abcdef01");
assert.equal(outcome.broken.ok, false);
assert.match(outcome.broken.error ?? "", /the dump failed/);
assert.equal(outcome.mail.ok, true);
assert.deepEqual(calls, [
"restic cat config",
"sh -c dump-it",
"restic backup --json --tag module=pg /",
"sh -c fail-it",
"restic backup --json --tag module=mail /",
"restic forget --prune --group-by host,tags --keep-daily 14 --keep-weekly 8 --keep-monthly 6",
]);
// Recorded, so `backed-up` and the missed-night check read it.
const nights = JSON.parse(readFileSync(join(where.state, "nights.json"), "utf8"));
assert.equal(nights.broken.ok, false);
assert.equal(nights.mail.ok, true);
});
test("a repository that exists and will not open is never replaced", async () => {
const where = place("# pg\npath /\n");
const calls: string[] = [];
const run: Runner = async (cmd, args) => {
calls.push(args.slice(5).join(" "));
if (args.includes("cat")) throw new Error("Fatal: wrong password or no key found");
return "";
};
await assert.rejects(new Backups(where, run, undefined, () => {}).backUp(), /cannot be opened, and is left as it is/);
assert.ok(!calls.includes("init"), "it made a new repository over one it could not open");
});
test("a declared directory that does not exist fails that module's night", async () => {
const where = place("# pg\npath /nowhere/at/all\n");
const run: Runner = async () => "";
const outcome = await new Backups(where, run, undefined, () => {}).backUp();
assert.equal(outcome.pg.ok, false);
assert.match(outcome.pg.error ?? "", /\/nowhere\/at\/all does not exist/);
});
test("a night is due at the hour today while it is ahead, else tomorrow; a missed one is noticed", () => {
const morning = new Date(2026, 9, 6, 1, 30);
assert.equal(nextNight(morning, 3).getTime(), new Date(2026, 9, 6, 3, 0).getTime());
const afternoon = new Date(2026, 9, 6, 15, 0);
assert.equal(nextNight(afternoon, 3).getTime(), new Date(2026, 9, 7, 3, 0).getTime());
assert.equal(missedANight({}, afternoon), true);
assert.equal(missedANight({ pg: { ok: true, at: new Date(2026, 9, 6, 3, 5).toISOString() } }, afternoon), false);
assert.equal(missedANight({ pg: { ok: true, at: new Date(2026, 9, 4, 3, 5).toISOString() } }, afternoon), true);
assert.equal(missedANight({ pg: { ok: false, at: new Date(2026, 9, 6, 3, 5).toISOString() } }, afternoon), true);
});
// The real thing, where restic is installed: a dump, a snapshot, a mistake, and a restore beside.
const hasRestic = (() => {
try {
execFileSync("restic", ["version"], { stdio: "ignore" });
return true;
} catch {
return false;
}
})();
test("with the real restic: a mistake is undone by a restore beside the live data", { skip: !hasRestic && "restic is not installed" }, async () => {
const root = mkdtempSync(join(tmpdir(), "restic-real-"));
const store = join(root, "store");
const dumps = join(root, "dumps");
mkdirSync(store);
mkdirSync(dumps);
writeFileSync(join(store, "mailbox"), "the only copy of a letter\n");
const where = place(`# mail\npath ${store}\n# pg\nrun echo 'every row' > ${dumps}/all.dump\npath ${dumps}\n`);
const backups = new Backups(where, undefined, () => new Date("2026-10-06T03:00:00Z"), () => {});
// escalated() would sudo; the test runs as whoever it is, against its own repository.
(backups as unknown as { run: Runner }).run = async (cmd, args) => execFileSync(cmd, args, { encoding: "utf8" });
const night = await backups.backUp();
assert.equal(night.mail.ok, true, night.mail.error);
assert.equal(night.pg.ok, true, night.pg.error);
assert.equal(readFileSync(join(dumps, "all.dump"), "utf8"), "every row\n");
// The mistake.
rmSync(join(store, "mailbox"));
const listed = await backups.backedUp();
assert.deepEqual(listed.map((m) => [m.module, m.restorePoints]), [["mail", 1], ["pg", 1]]);
const restored = await backups.restore("mail");
assert.equal(restored.restored.length, 1);
assert.match(restored.restored[0], /store\.restored-20261006-030000$/);
assert.equal(readFileSync(join(restored.restored[0], "mailbox"), "utf8"), "the only copy of a letter\n");
assert.ok(!existsSync(join(store, "mailbox")), "the restore wrote into the live directory");
// Never over anything: the same restore again finds its target taken.
await assert.rejects(backups.restore("mail"), /already exists; nothing is restored over anything/);
});