jschoubben
32cd92baeb
ADR 0214 / to-be 43. restic keeps one repository per machine and takes a nightly snapshot per module — 14 daily, 8 weekly, 6 monthly — and restores beside the live data, never over it. postgres, mssql and mongodb contribute a consistent dump; minio, influxdb, the vault, mailu, gitea and nextcloud the directories that hold their data.
79 lines
3.7 KiB
TypeScript
79 lines
3.7 KiB
TypeScript
// The machine's backups: the node-backup seat's three verbs — what is backed up, take one now,
|
|
// restore beside the live data — and the night that runs without anyone asking (novox/hq ADR 0214,
|
|
// to-be 43). What is backed up is composed by the mesh from the modules the machine runs; this code
|
|
// only runs it.
|
|
|
|
import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools";
|
|
import { Backups, missedANight, nextNight, whereFromEnv } from "../client.js";
|
|
|
|
export function getSeatVerbs(backups: Backups): ToolDefinition[] {
|
|
return [
|
|
{
|
|
name: "backed-up",
|
|
description:
|
|
"What this machine backs up: each module, what it declared, its last good night, how many restore points are kept.",
|
|
input: { module: { type: "string", description: "one module (optional)" } },
|
|
run: async (args) => backups.backedUp(args.module ? String(args.module) : undefined),
|
|
},
|
|
{
|
|
name: "now",
|
|
description:
|
|
"Take a backup now, of one module or of every module on this machine — before a migration, a retirement or anything else that could go wrong. Answers when it has started; `backed-up` says how it went.",
|
|
input: { module: { type: "string", description: "one module (optional)" } },
|
|
run: async (args) => {
|
|
const only = args.module ? String(args.module) : undefined;
|
|
// A night of a large store outlasts any call; it is started, and its outcome recorded.
|
|
backups.backUp(only).catch((err) => console.error(`[restic] a backup asked for now failed: ${(err as Error).message}`));
|
|
return { started: only ?? "every module on this machine", follow: "backed-up" };
|
|
},
|
|
},
|
|
{
|
|
name: "restore",
|
|
description:
|
|
"Restore one module's data from a restore point BESIDE the live data, never over it: each directory as <path>.restored-<date>. Swapping it in is a person's act.",
|
|
input: {
|
|
module: { type: "string", description: "the module" },
|
|
snapshot: { type: "string", description: "the restore point (the newest when omitted)" },
|
|
path: { type: "string", description: "one of the module's directories (all of them when omitted)" },
|
|
},
|
|
run: async (args) =>
|
|
backups.restore(String(args.module ?? ""), args.snapshot ? String(args.snapshot) : undefined, args.path ? String(args.path) : undefined),
|
|
},
|
|
];
|
|
}
|
|
|
|
const backups = new Backups(whereFromEnv());
|
|
registerModuleTools("node-backup", () => getSeatVerbs(backups));
|
|
|
|
// The night. At the hour, every module; and at start, if a night was missed — the machine was off
|
|
// or this module was not running at the hour — one now rather than a day later.
|
|
const HOUR = Number(process.env.MESH_BACKUP_HOUR ?? "3");
|
|
|
|
async function night(): Promise<void> {
|
|
try {
|
|
const outcome = await backups.backUp();
|
|
const failed = Object.entries(outcome).filter(([, n]) => !n.ok).map(([m]) => m);
|
|
if (failed.length > 0) console.error(`[restic] the night left ${failed.join(", ")} without a backup`);
|
|
// Sundays, the repository's own integrity with a sample of the data read back.
|
|
if (new Date().getDay() === 0) {
|
|
await backups.check().then(
|
|
() => console.error("[restic] the repository checks out"),
|
|
(err) => console.error(`[restic] the repository does NOT check out: ${(err as Error).message}`),
|
|
);
|
|
}
|
|
} catch (err) {
|
|
console.error(`[restic] the night did not run: ${(err as Error).message}`);
|
|
}
|
|
}
|
|
|
|
function schedule(): void {
|
|
const at = nextNight(new Date(), HOUR);
|
|
setTimeout(() => void night().finally(schedule), at.getTime() - Date.now());
|
|
}
|
|
|
|
if (missedANight(backups.nights(), new Date())) {
|
|
// Not at once: a machine just started has its stores still coming up.
|
|
setTimeout(() => void night(), 10 * 60 * 1000);
|
|
}
|
|
schedule();
|