restic: the holder in Go
Go is the default for new module code; the holder is one binary like the licence manager, serving the seat's verbs over the SDK and running the nights beside them. Same behaviour and tests.
This commit is contained in:
@@ -1,312 +0,0 @@
|
||||
// restic's own code, in the module (novox/hq ADR 0039): the machine's backups (ADR 0214, to-be 43).
|
||||
//
|
||||
// The mesh composes what to back up: every module on the machine contributes `backup` lines to the
|
||||
// node-backup seat, and the mesh writes them, each module's under a `# <module>` line and with its
|
||||
// directories already filled, into one file this module reads. Two kinds of line:
|
||||
//
|
||||
// run <shell command> run as root before the module's snapshot — a consistent dump of a store
|
||||
// path <directory> a directory the module's snapshot keeps
|
||||
//
|
||||
// Each module gets one snapshot a night, tagged with its name, so a module is listed, kept and
|
||||
// restored on its own. Everything lands in one repository on the machine — deduplicated, so every
|
||||
// night is a complete restore point and only what changed costs space — and is thinned to 14 daily,
|
||||
// 8 weekly and 6 monthly. Against mistakes, not disasters: nothing leaves the machine.
|
||||
//
|
||||
// Root's: the dumps read every store and the repository holds every module's data, and the runtime
|
||||
// loading this bundle runs as the operator's account, so restic and the run lines go through sudo
|
||||
// without a prompt where the account is not root — fail2ban's way (ADR 0175 §4).
|
||||
|
||||
import { execFile } from "node:child_process";
|
||||
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
const execFileP = promisify(execFile);
|
||||
|
||||
/** A command runner, so the backups can be tested without restic or a store. */
|
||||
export type Runner = (cmd: string, args: string[]) => Promise<string>;
|
||||
|
||||
/** The command as it is run: as given when this process is root, else through sudo without a prompt. */
|
||||
export function escalated(cmd: string, args: string[], uid: number | undefined = process.getuid?.()): [string, string[]] {
|
||||
if (uid === 0) return [cmd, args];
|
||||
return ["sudo", ["-n", cmd, ...args]];
|
||||
}
|
||||
|
||||
export const execRunner: Runner = async (cmd, args) => {
|
||||
const [program, argv] = escalated(cmd, args);
|
||||
try {
|
||||
// A night's dump of a large store takes a while; six hours is a dump that will not finish.
|
||||
const { stdout } = await execFileP(program, argv, { maxBuffer: 256 * 1024 * 1024, timeout: 6 * 3600 * 1000 });
|
||||
return stdout;
|
||||
} catch (err) {
|
||||
const e = err as { code?: string | number; stderr?: string; stdout?: string; message?: string };
|
||||
const said = `${e.stderr ?? ""}`.trim() || `${e.stdout ?? ""}`.trim();
|
||||
if (program === "sudo" && /^sudo:/m.test(said)) {
|
||||
throw new Error(`${cmd} needs root and the runtime's account may not run it without a prompt: ${said}`);
|
||||
}
|
||||
const lines = said.split("\n").map((l) => l.trim()).filter(Boolean);
|
||||
throw new Error(lines.length ? lines.slice(-3).join(" / ") : (e.message ?? `${cmd} failed`));
|
||||
}
|
||||
};
|
||||
|
||||
/** What one module declared. */
|
||||
export interface Declared {
|
||||
module: string;
|
||||
runs: string[];
|
||||
paths: string[];
|
||||
}
|
||||
|
||||
/** The composed file, read into each module's declaration, in the order the mesh wrote them. A line
|
||||
* before any module, a comment that is not a module's name, or a blank, is nothing. */
|
||||
export function parseDeclared(text: string): Declared[] {
|
||||
const out: Declared[] = [];
|
||||
let current: Declared | undefined;
|
||||
for (const raw of text.split("\n")) {
|
||||
const line = raw.trim();
|
||||
if (line === "") continue;
|
||||
const header = /^#\s*([a-z0-9][a-z0-9-]*)$/.exec(line);
|
||||
if (header) {
|
||||
current = { module: header[1], runs: [], paths: [] };
|
||||
out.push(current);
|
||||
continue;
|
||||
}
|
||||
if (line.startsWith("#") || !current) continue;
|
||||
const [kind, ...rest] = line.split(/\s+/);
|
||||
const value = line.slice(kind.length).trim();
|
||||
if (kind === "run" && value) current.runs.push(value);
|
||||
else if (kind === "path" && rest.length === 1 && value.startsWith("/")) current.paths.push(value);
|
||||
else throw new Error(`${current.module} contributes a backup line this holder does not read: ${line}`);
|
||||
}
|
||||
return out.filter((d) => d.runs.length > 0 || d.paths.length > 0);
|
||||
}
|
||||
|
||||
/** One restore point, as restic lists it. */
|
||||
export interface Snapshot {
|
||||
id: string;
|
||||
short_id: string;
|
||||
time: string;
|
||||
paths: string[];
|
||||
tags?: string[];
|
||||
hostname: string;
|
||||
}
|
||||
|
||||
/** How one module's last night went. */
|
||||
export interface Night {
|
||||
ok: boolean;
|
||||
at: string;
|
||||
snapshot?: string;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export const KEEP = { daily: 14, weekly: 8, monthly: 6 };
|
||||
|
||||
export function tagOf(module: string): string {
|
||||
return `module=${module}`;
|
||||
}
|
||||
|
||||
export interface Where {
|
||||
declared: string;
|
||||
repository: string;
|
||||
passwordFile: string;
|
||||
state: string;
|
||||
}
|
||||
|
||||
export function whereFromEnv(env: NodeJS.ProcessEnv = process.env): Where {
|
||||
const need = (k: string) => {
|
||||
const v = env[k];
|
||||
if (!v) throw new Error(`${k} is not set; the mesh gives it to this module's tools`);
|
||||
return v;
|
||||
};
|
||||
return {
|
||||
declared: need("MESH_BACKUP_DECLARED"),
|
||||
repository: need("MESH_BACKUP_REPOSITORY"),
|
||||
passwordFile: need("MESH_BACKUP_PASSWORD_FILE"),
|
||||
state: need("MESH_BACKUP_STATE"),
|
||||
};
|
||||
}
|
||||
|
||||
export class Backups {
|
||||
private busy: Promise<unknown> = Promise.resolve();
|
||||
readonly where: Where;
|
||||
private run: Runner;
|
||||
private readonly now: () => Date;
|
||||
private readonly say: (line: string) => void;
|
||||
|
||||
constructor(
|
||||
where: Where,
|
||||
run: Runner = execRunner,
|
||||
now: () => Date = () => new Date(),
|
||||
say: (line: string) => void = (l) => console.error(`[restic] ${l}`),
|
||||
) {
|
||||
this.where = where;
|
||||
this.run = run;
|
||||
this.now = now;
|
||||
this.say = say;
|
||||
}
|
||||
|
||||
private restic(args: string[]): Promise<string> {
|
||||
return this.run("restic", ["--repo", this.where.repository, "--password-file", this.where.passwordFile, "--no-cache", ...args]);
|
||||
}
|
||||
|
||||
/** One thing at a time: two nights, or a night and a restore, never share a dump. */
|
||||
private serial<T>(work: () => Promise<T>): Promise<T> {
|
||||
const next = this.busy.then(work, work);
|
||||
this.busy = next.catch(() => undefined);
|
||||
return next;
|
||||
}
|
||||
|
||||
declared(): Declared[] {
|
||||
return parseDeclared(readFileSync(this.where.declared, "utf8"));
|
||||
}
|
||||
|
||||
private nightsFile(): string {
|
||||
return join(this.where.state, "nights.json");
|
||||
}
|
||||
|
||||
nights(): Record<string, Night> {
|
||||
try {
|
||||
return JSON.parse(readFileSync(this.nightsFile(), "utf8")) as Record<string, Night>;
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
private record(module: string, night: Night): void {
|
||||
const all = this.nights();
|
||||
all[module] = night;
|
||||
writeFileSync(this.nightsFile(), JSON.stringify(all, null, 2) + "\n", { mode: 0o600 });
|
||||
}
|
||||
|
||||
/** The repository, made the first time. A repository that exists and cannot be opened is said,
|
||||
* never replaced: replacing it would discard every restore point to fix a password. */
|
||||
async ensureRepository(): Promise<void> {
|
||||
try {
|
||||
await this.restic(["cat", "config"]);
|
||||
} catch (err) {
|
||||
const why = String((err as Error).message);
|
||||
if (!/does not exist|unable to open config file|Is there a repository at the following location/i.test(why)) {
|
||||
throw new Error(`the repository at ${this.where.repository} cannot be opened, and is left as it is: ${why}`);
|
||||
}
|
||||
this.say(`no repository at ${this.where.repository}; making one`);
|
||||
await this.restic(["init"]);
|
||||
}
|
||||
}
|
||||
|
||||
/** One module's night: its run lines, then one snapshot of its paths. A failure is that module's. */
|
||||
private async one(d: Declared): Promise<Night> {
|
||||
const at = this.now().toISOString();
|
||||
try {
|
||||
for (const command of d.runs) {
|
||||
await this.run("sh", ["-c", command]);
|
||||
}
|
||||
const missing = d.paths.filter((p) => !existsSync(p));
|
||||
if (missing.length > 0) throw new Error(`${missing.join(", ")} does not exist`);
|
||||
if (d.paths.length === 0) throw new Error("it runs a dump and names no directory to keep it from");
|
||||
const out = await this.restic(["backup", "--json", "--tag", tagOf(d.module), ...d.paths]);
|
||||
const summary = out
|
||||
.split("\n")
|
||||
.map((l) => { try { return JSON.parse(l) as { message_type?: string; snapshot_id?: string }; } catch { return {}; } })
|
||||
.find((m) => m.message_type === "summary");
|
||||
const night: Night = { ok: true, at, snapshot: summary?.snapshot_id?.slice(0, 8) };
|
||||
this.say(`${d.module}: backed up (${night.snapshot ?? "no snapshot id reported"})`);
|
||||
return night;
|
||||
} catch (err) {
|
||||
const night: Night = { ok: false, at, error: String((err as Error).message) };
|
||||
this.say(`${d.module}: NOT backed up: ${night.error}`);
|
||||
return night;
|
||||
}
|
||||
}
|
||||
|
||||
/** A night: every module, or one, then the rotation. Returns each module's outcome. */
|
||||
backUp(only?: string): Promise<Record<string, Night>> {
|
||||
return this.serial(async () => {
|
||||
await this.ensureRepository();
|
||||
const all = this.declared();
|
||||
const chosen = only ? all.filter((d) => d.module === only) : all;
|
||||
if (only && chosen.length === 0) {
|
||||
throw new Error(`${only} declares nothing to back up on this machine; it backs up ${all.map((d) => d.module).join(", ") || "nothing"}`);
|
||||
}
|
||||
const outcome: Record<string, Night> = {};
|
||||
for (const d of chosen) {
|
||||
outcome[d.module] = await this.one(d);
|
||||
this.record(d.module, outcome[d.module]);
|
||||
}
|
||||
await this.restic([
|
||||
"forget", "--prune", "--group-by", "host,tags",
|
||||
"--keep-daily", String(KEEP.daily), "--keep-weekly", String(KEEP.weekly), "--keep-monthly", String(KEEP.monthly),
|
||||
]).catch((err) => this.say(`thinning the restore points failed, and every one is kept: ${(err as Error).message}`));
|
||||
return outcome;
|
||||
});
|
||||
}
|
||||
|
||||
async snapshots(module?: string): Promise<Snapshot[]> {
|
||||
const args = ["snapshots", "--json"];
|
||||
if (module) args.push("--tag", tagOf(module));
|
||||
return JSON.parse((await this.restic(args)) || "[]") as Snapshot[];
|
||||
}
|
||||
|
||||
/** What is backed up here: each module, what it declared, its last night and its restore points. */
|
||||
async backedUp(module?: string) {
|
||||
const nights = this.nights();
|
||||
const snaps = await this.snapshots(module).catch(() => [] as Snapshot[]);
|
||||
return this.declared()
|
||||
.filter((d) => !module || d.module === module)
|
||||
.map((d) => {
|
||||
const mine = snaps.filter((s) => (s.tags ?? []).includes(tagOf(d.module)));
|
||||
return {
|
||||
module: d.module,
|
||||
runs: d.runs.length,
|
||||
paths: d.paths,
|
||||
lastNight: nights[d.module] ?? null,
|
||||
restorePoints: mine.length,
|
||||
newest: mine.length ? { snapshot: mine[mine.length - 1].short_id, at: mine[mine.length - 1].time } : null,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/** A module's data from a restore point, BESIDE the live data: each directory as
|
||||
* <path>.restored-<stamp>. A target that already exists is refused, never overwritten. */
|
||||
restore(module: string, snapshot?: string, path?: string) {
|
||||
return this.serial(async () => {
|
||||
const mine = await this.snapshots(module);
|
||||
if (mine.length === 0) throw new Error(`${module} has no restore point on this machine`);
|
||||
const chosen = snapshot ? mine.find((s) => s.id.startsWith(snapshot) || s.short_id === snapshot) : mine[mine.length - 1];
|
||||
if (!chosen) {
|
||||
throw new Error(`${module} has no restore point ${snapshot}; it has ${mine.map((s) => `${s.short_id} (${s.time})`).join(", ")}`);
|
||||
}
|
||||
const paths = path ? chosen.paths.filter((p) => p === path) : chosen.paths;
|
||||
if (paths.length === 0) throw new Error(`restore point ${chosen.short_id} of ${module} holds ${chosen.paths.join(", ")}, not ${path}`);
|
||||
const stamp = this.now().toISOString().replace(/[-:]/g, "").replace("T", "-").slice(0, 15);
|
||||
const restored: string[] = [];
|
||||
for (const p of paths) {
|
||||
const target = `${p}.restored-${stamp}`;
|
||||
if (existsSync(target)) throw new Error(`${target} already exists; nothing is restored over anything`);
|
||||
await this.restic(["restore", `${chosen.id}:${p}`, "--target", target]);
|
||||
restored.push(target);
|
||||
this.say(`${module}: restored ${p} from ${chosen.short_id} to ${target}`);
|
||||
}
|
||||
return { module, from: { snapshot: chosen.short_id, at: chosen.time }, restored, live: "untouched — swapping it in is a person's act" };
|
||||
});
|
||||
}
|
||||
|
||||
/** The weekly look at the repository's own integrity, with a sample of the data read back. */
|
||||
check(): Promise<string> {
|
||||
return this.serial(() => this.restic(["check", "--read-data-subset", "5%"]));
|
||||
}
|
||||
}
|
||||
|
||||
/** When the next night is due: the given hour, local time, today if it is still ahead, else tomorrow. */
|
||||
export function nextNight(now: Date, hour: number): Date {
|
||||
const next = new Date(now);
|
||||
next.setHours(hour, 0, 0, 0);
|
||||
if (next.getTime() <= now.getTime()) next.setDate(next.getDate() + 1);
|
||||
return next;
|
||||
}
|
||||
|
||||
/** Whether a night was missed: the newest good night of any module is older than a day and a bit —
|
||||
* the machine was off, or this module was not running, at the hour. */
|
||||
export function missedANight(nights: Record<string, Night>, now: Date): boolean {
|
||||
const good = Object.values(nights).filter((n) => n.ok).map((n) => Date.parse(n.at));
|
||||
if (good.length === 0) return true;
|
||||
return now.getTime() - Math.max(...good) > 26 * 3600 * 1000;
|
||||
}
|
||||
@@ -0,0 +1,472 @@
|
||||
// The machine's backups (novox/hq ADR 0214, to-be 43).
|
||||
//
|
||||
// The mesh composes what to back up: every module on the machine contributes `backup` lines to the
|
||||
// node-backup seat, and the mesh writes them, each module's under a `# <module>` line and with its
|
||||
// directories already filled, into one file this module reads. Two kinds of line:
|
||||
//
|
||||
// run <shell command> run as root before the module's snapshot — a consistent dump of a store
|
||||
// path <directory> a directory the module's snapshot keeps
|
||||
//
|
||||
// Each module gets one snapshot a night, tagged with its name, so a module is listed, kept and
|
||||
// restored on its own. Everything lands in one repository on the machine — deduplicated, so every
|
||||
// night is a complete restore point and only what changed costs space — and is thinned to 14 daily,
|
||||
// 8 weekly and 6 monthly. Against mistakes, not disasters: nothing leaves the machine.
|
||||
//
|
||||
// Root's: the dumps read every store and the repository holds every module's data, and the runtime
|
||||
// launching this binary runs as the operator's account, so restic and the run lines go through sudo
|
||||
// without a prompt where the account is not root (ADR 0175 §4).
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Runner runs one command and answers what it printed, so the backups can be tested without restic
|
||||
// or a store.
|
||||
type Runner func(ctx context.Context, name string, args ...string) (string, error)
|
||||
|
||||
// escalated is the command as it is run: as given when this process is root, else through sudo
|
||||
// without a prompt.
|
||||
func escalated(uid int, name string, args []string) (string, []string) {
|
||||
if uid == 0 {
|
||||
return name, args
|
||||
}
|
||||
return "sudo", append([]string{"-n", name}, args...)
|
||||
}
|
||||
|
||||
// execRunner runs it for real. A night's dump of a large store takes a while; six hours is a dump
|
||||
// that will not finish.
|
||||
func execRunner(ctx context.Context, name string, args ...string) (string, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 6*time.Hour)
|
||||
defer cancel()
|
||||
program, argv := escalated(os.Getuid(), name, args)
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd := exec.CommandContext(ctx, program, argv...)
|
||||
cmd.Stdout, cmd.Stderr = &stdout, &stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
said := strings.TrimSpace(stderr.String())
|
||||
if said == "" {
|
||||
said = strings.TrimSpace(stdout.String())
|
||||
}
|
||||
if program == "sudo" && strings.HasPrefix(said, "sudo:") {
|
||||
return "", fmt.Errorf("%s needs root and the runtime's account may not run it without a prompt: %s", name, said)
|
||||
}
|
||||
lines := strings.Split(said, "\n")
|
||||
if len(lines) > 3 {
|
||||
lines = lines[len(lines)-3:]
|
||||
}
|
||||
if said == "" {
|
||||
return "", fmt.Errorf("%s: %w", name, err)
|
||||
}
|
||||
return "", errors.New(strings.Join(lines, " / "))
|
||||
}
|
||||
return stdout.String(), nil
|
||||
}
|
||||
|
||||
// Declared is what one module declared.
|
||||
type Declared struct {
|
||||
Module string `json:"module"`
|
||||
Runs []string `json:"runs"`
|
||||
Paths []string `json:"paths"`
|
||||
}
|
||||
|
||||
var moduleHeader = regexp.MustCompile(`^#\s*([a-z0-9][a-z0-9-]*)$`)
|
||||
|
||||
// parseDeclared reads the composed file into each module's declaration, in the order the mesh wrote
|
||||
// them. A line before any module, a comment that is not a module's name, or a blank, is nothing; a
|
||||
// line this holder does not read is refused, naming the module, rather than skipped.
|
||||
func parseDeclared(text string) ([]Declared, error) {
|
||||
var out []Declared
|
||||
current := -1
|
||||
for _, raw := range strings.Split(text, "\n") {
|
||||
line := strings.TrimSpace(raw)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
if m := moduleHeader.FindStringSubmatch(line); m != nil {
|
||||
out = append(out, Declared{Module: m[1]})
|
||||
current = len(out) - 1
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(line, "#") || current < 0 {
|
||||
continue
|
||||
}
|
||||
kind, value, _ := strings.Cut(line, " ")
|
||||
value = strings.TrimSpace(value)
|
||||
d := &out[current]
|
||||
switch {
|
||||
case kind == "run" && value != "":
|
||||
d.Runs = append(d.Runs, value)
|
||||
case kind == "path" && strings.HasPrefix(value, "/") && !strings.ContainsAny(value, " \t"):
|
||||
d.Paths = append(d.Paths, value)
|
||||
default:
|
||||
return nil, fmt.Errorf("%s contributes a backup line this holder does not read: %s", d.Module, line)
|
||||
}
|
||||
}
|
||||
kept := out[:0]
|
||||
for _, d := range out {
|
||||
if len(d.Runs) > 0 || len(d.Paths) > 0 {
|
||||
kept = append(kept, d)
|
||||
}
|
||||
}
|
||||
return kept, nil
|
||||
}
|
||||
|
||||
// Snapshot is one restore point, as restic lists it.
|
||||
type Snapshot struct {
|
||||
ID string `json:"id"`
|
||||
ShortID string `json:"short_id"`
|
||||
Time time.Time `json:"time"`
|
||||
Paths []string `json:"paths"`
|
||||
Tags []string `json:"tags"`
|
||||
Hostname string `json:"hostname"`
|
||||
}
|
||||
|
||||
// Night is how one module's last night went.
|
||||
type Night struct {
|
||||
OK bool `json:"ok"`
|
||||
At time.Time `json:"at"`
|
||||
Snapshot string `json:"snapshot,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// Keep is the rotation (novox/hq ADR 0214).
|
||||
var Keep = struct{ Daily, Weekly, Monthly int }{14, 8, 6}
|
||||
|
||||
func tagOf(module string) string { return "module=" + module }
|
||||
|
||||
// Where is where the mesh put this module's things.
|
||||
type Where struct {
|
||||
Declared, Repository, PasswordFile, State string
|
||||
}
|
||||
|
||||
func whereFromEnv() (Where, error) {
|
||||
var missing []string
|
||||
get := func(k string) string {
|
||||
v := os.Getenv(k)
|
||||
if v == "" {
|
||||
missing = append(missing, k)
|
||||
}
|
||||
return v
|
||||
}
|
||||
w := Where{
|
||||
Declared: get("MESH_BACKUP_DECLARED"),
|
||||
Repository: get("MESH_BACKUP_REPOSITORY"),
|
||||
PasswordFile: get("MESH_BACKUP_PASSWORD_FILE"),
|
||||
State: get("MESH_BACKUP_STATE"),
|
||||
}
|
||||
if len(missing) > 0 {
|
||||
return w, fmt.Errorf("%s not set; the mesh gives them to this module", strings.Join(missing, ", "))
|
||||
}
|
||||
return w, nil
|
||||
}
|
||||
|
||||
// Backups is the machine's backups. One thing at a time: two nights, or a night and a restore, never
|
||||
// share a dump.
|
||||
type Backups struct {
|
||||
Where Where
|
||||
Run Runner
|
||||
Now func() time.Time
|
||||
Say func(format string, args ...any)
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func (b *Backups) restic(ctx context.Context, args ...string) (string, error) {
|
||||
return b.Run(ctx, "restic", append([]string{"--repo", b.Where.Repository, "--password-file", b.Where.PasswordFile, "--no-cache"}, args...)...)
|
||||
}
|
||||
|
||||
// Declared is what the modules on this machine declared.
|
||||
func (b *Backups) Declared() ([]Declared, error) {
|
||||
raw, err := os.ReadFile(b.Where.Declared)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return parseDeclared(string(raw))
|
||||
}
|
||||
|
||||
func (b *Backups) nightsFile() string { return filepath.Join(b.Where.State, "nights.json") }
|
||||
|
||||
// Nights is how each module's last night went.
|
||||
func (b *Backups) Nights() map[string]Night {
|
||||
nights := map[string]Night{}
|
||||
if raw, err := os.ReadFile(b.nightsFile()); err == nil {
|
||||
_ = json.Unmarshal(raw, &nights)
|
||||
}
|
||||
return nights
|
||||
}
|
||||
|
||||
func (b *Backups) record(module string, n Night) {
|
||||
nights := b.Nights()
|
||||
nights[module] = n
|
||||
raw, _ := json.MarshalIndent(nights, "", " ")
|
||||
if err := os.WriteFile(b.nightsFile(), append(raw, '\n'), 0o600); err != nil {
|
||||
b.Say("recording %s's night failed: %v", module, err)
|
||||
}
|
||||
}
|
||||
|
||||
var noRepository = regexp.MustCompile(`(?i)does not exist|unable to open config file|Is there a repository at the following location`)
|
||||
|
||||
// ensureRepository makes the repository the first time. One that exists and cannot be opened is
|
||||
// said, never replaced: replacing it would discard every restore point to fix a password.
|
||||
func (b *Backups) ensureRepository(ctx context.Context) error {
|
||||
_, err := b.restic(ctx, "cat", "config")
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if !noRepository.MatchString(err.Error()) {
|
||||
return fmt.Errorf("the repository at %s cannot be opened, and is left as it is: %v", b.Where.Repository, err)
|
||||
}
|
||||
b.Say("no repository at %s; making one", b.Where.Repository)
|
||||
_, err = b.restic(ctx, "init")
|
||||
return err
|
||||
}
|
||||
|
||||
// one is one module's night: its run lines, then one snapshot of its paths. A failure is that
|
||||
// module's alone.
|
||||
func (b *Backups) one(ctx context.Context, d Declared) Night {
|
||||
n := Night{At: b.Now().UTC()}
|
||||
fail := func(err error) Night {
|
||||
n.Error = err.Error()
|
||||
b.Say("%s: NOT backed up: %s", d.Module, n.Error)
|
||||
return n
|
||||
}
|
||||
for _, command := range d.Runs {
|
||||
if _, err := b.Run(ctx, "sh", "-c", command); err != nil {
|
||||
return fail(err)
|
||||
}
|
||||
}
|
||||
if len(d.Paths) == 0 {
|
||||
return fail(errors.New("it runs a dump and names no directory to keep it from"))
|
||||
}
|
||||
var missing []string
|
||||
for _, p := range d.Paths {
|
||||
if _, err := os.Stat(p); err != nil {
|
||||
missing = append(missing, p)
|
||||
}
|
||||
}
|
||||
if len(missing) > 0 {
|
||||
return fail(fmt.Errorf("%s does not exist", strings.Join(missing, ", ")))
|
||||
}
|
||||
out, err := b.restic(ctx, append([]string{"backup", "--json", "--tag", tagOf(d.Module)}, d.Paths...)...)
|
||||
if err != nil {
|
||||
return fail(err)
|
||||
}
|
||||
scanner := bufio.NewScanner(strings.NewReader(out))
|
||||
scanner.Buffer(make([]byte, 1024*1024), 16*1024*1024)
|
||||
for scanner.Scan() {
|
||||
var m struct {
|
||||
MessageType string `json:"message_type"`
|
||||
SnapshotID string `json:"snapshot_id"`
|
||||
}
|
||||
if json.Unmarshal(scanner.Bytes(), &m) == nil && m.MessageType == "summary" && len(m.SnapshotID) >= 8 {
|
||||
n.Snapshot = m.SnapshotID[:8]
|
||||
}
|
||||
}
|
||||
n.OK = true
|
||||
b.Say("%s: backed up (%s)", d.Module, n.Snapshot)
|
||||
return n
|
||||
}
|
||||
|
||||
// BackUp is a night: every module, or one, then the rotation. It answers each module's outcome.
|
||||
func (b *Backups) BackUp(ctx context.Context, only string) (map[string]Night, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if err := b.ensureRepository(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
all, err := b.Declared()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
chosen := all
|
||||
if only != "" {
|
||||
chosen = nil
|
||||
var names []string
|
||||
for _, d := range all {
|
||||
names = append(names, d.Module)
|
||||
if d.Module == only {
|
||||
chosen = append(chosen, d)
|
||||
}
|
||||
}
|
||||
if len(chosen) == 0 {
|
||||
return nil, fmt.Errorf("%s declares nothing to back up on this machine; it backs up %s", only, orNothing(names))
|
||||
}
|
||||
}
|
||||
outcome := map[string]Night{}
|
||||
for _, d := range chosen {
|
||||
outcome[d.Module] = b.one(ctx, d)
|
||||
b.record(d.Module, outcome[d.Module])
|
||||
}
|
||||
if _, err := b.restic(ctx, "forget", "--prune", "--group-by", "host,tags",
|
||||
"--keep-daily", fmt.Sprint(Keep.Daily), "--keep-weekly", fmt.Sprint(Keep.Weekly), "--keep-monthly", fmt.Sprint(Keep.Monthly)); err != nil {
|
||||
b.Say("thinning the restore points failed, and every one is kept: %v", err)
|
||||
}
|
||||
return outcome, nil
|
||||
}
|
||||
|
||||
func orNothing(names []string) string {
|
||||
if len(names) == 0 {
|
||||
return "nothing"
|
||||
}
|
||||
return strings.Join(names, ", ")
|
||||
}
|
||||
|
||||
// Snapshots is the restore points, of one module or all.
|
||||
func (b *Backups) Snapshots(ctx context.Context, module string) ([]Snapshot, error) {
|
||||
args := []string{"snapshots", "--json"}
|
||||
if module != "" {
|
||||
args = append(args, "--tag", tagOf(module))
|
||||
}
|
||||
out, err := b.restic(ctx, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var snaps []Snapshot
|
||||
if strings.TrimSpace(out) == "" {
|
||||
return nil, nil
|
||||
}
|
||||
if err := json.Unmarshal([]byte(out), &snaps); err != nil {
|
||||
return nil, fmt.Errorf("restic listed its snapshots in a form this holder does not read: %v", err)
|
||||
}
|
||||
return snaps, nil
|
||||
}
|
||||
|
||||
// ModuleBackups is what `backed-up` says about one module.
|
||||
type ModuleBackups struct {
|
||||
Module string `json:"module"`
|
||||
Runs int `json:"runs"`
|
||||
Paths []string `json:"paths"`
|
||||
LastNight *Night `json:"lastNight"`
|
||||
RestorePoints int `json:"restorePoints"`
|
||||
Newest *Snapshot `json:"newest,omitempty"`
|
||||
}
|
||||
|
||||
// BackedUp is what is backed up here: each module, what it declared, its last night and its restore
|
||||
// points.
|
||||
func (b *Backups) BackedUp(ctx context.Context, module string) ([]ModuleBackups, error) {
|
||||
declared, err := b.Declared()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nights := b.Nights()
|
||||
snaps, _ := b.Snapshots(ctx, module)
|
||||
out := []ModuleBackups{}
|
||||
for _, d := range declared {
|
||||
if module != "" && d.Module != module {
|
||||
continue
|
||||
}
|
||||
m := ModuleBackups{Module: d.Module, Runs: len(d.Runs), Paths: d.Paths}
|
||||
if n, ok := nights[d.Module]; ok {
|
||||
m.LastNight = &n
|
||||
}
|
||||
for _, s := range snaps {
|
||||
if slices.Contains(s.Tags, tagOf(d.Module)) {
|
||||
m.RestorePoints++
|
||||
newest := s
|
||||
m.Newest = &newest
|
||||
}
|
||||
}
|
||||
out = append(out, m)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Restored is what a restore put where.
|
||||
type Restored struct {
|
||||
Module string `json:"module"`
|
||||
From Snapshot `json:"from"`
|
||||
Restored []string `json:"restored"`
|
||||
Live string `json:"live"`
|
||||
}
|
||||
|
||||
// Restore puts a module's data from a restore point BESIDE the live data: each directory as
|
||||
// <path>.restored-<stamp>. A target that already exists is refused, never overwritten.
|
||||
func (b *Backups) Restore(ctx context.Context, module, snapshot, path string) (*Restored, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
mine, err := b.Snapshots(ctx, module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(mine) == 0 {
|
||||
return nil, fmt.Errorf("%s has no restore point on this machine", module)
|
||||
}
|
||||
chosen := mine[len(mine)-1]
|
||||
if snapshot != "" {
|
||||
found := false
|
||||
var listed []string
|
||||
for _, s := range mine {
|
||||
listed = append(listed, fmt.Sprintf("%s (%s)", s.ShortID, s.Time.Format(time.RFC3339)))
|
||||
if s.ShortID == snapshot || strings.HasPrefix(s.ID, snapshot) {
|
||||
chosen, found = s, true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("%s has no restore point %s; it has %s", module, snapshot, strings.Join(listed, ", "))
|
||||
}
|
||||
}
|
||||
paths := chosen.Paths
|
||||
if path != "" {
|
||||
if !slices.Contains(chosen.Paths, path) {
|
||||
return nil, fmt.Errorf("restore point %s of %s holds %s, not %s", chosen.ShortID, module, strings.Join(chosen.Paths, ", "), path)
|
||||
}
|
||||
paths = []string{path}
|
||||
}
|
||||
stamp := b.Now().UTC().Format("20060102-150405")
|
||||
r := &Restored{Module: module, From: chosen, Live: "untouched — swapping it in is a person's act"}
|
||||
for _, p := range paths {
|
||||
target := p + ".restored-" + stamp
|
||||
if _, err := os.Stat(target); err == nil {
|
||||
return nil, fmt.Errorf("%s already exists; nothing is restored over anything", target)
|
||||
}
|
||||
if _, err := b.restic(ctx, "restore", chosen.ID+":"+p, "--target", target); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r.Restored = append(r.Restored, target)
|
||||
b.Say("%s: restored %s from %s to %s", module, p, chosen.ShortID, target)
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// Check is the weekly look at the repository's own integrity, with a sample of the data read back.
|
||||
func (b *Backups) Check(ctx context.Context) error {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
_, err := b.restic(ctx, "check", "--read-data-subset", "5%")
|
||||
return err
|
||||
}
|
||||
|
||||
// nextNight is when the next night is due: the given hour, local time, today while it is still
|
||||
// ahead, else tomorrow.
|
||||
func nextNight(now time.Time, hour int) time.Time {
|
||||
next := time.Date(now.Year(), now.Month(), now.Day(), hour, 0, 0, 0, now.Location())
|
||||
if !next.After(now) {
|
||||
next = next.AddDate(0, 0, 1)
|
||||
}
|
||||
return next
|
||||
}
|
||||
|
||||
// missedANight is whether a night was missed: the newest good night of any module is older than a
|
||||
// day and a bit — the machine was off, or this module was not running, at the hour.
|
||||
func missedANight(nights map[string]Night, now time.Time) bool {
|
||||
var newest time.Time
|
||||
for _, n := range nights {
|
||||
if n.OK && n.At.After(newest) {
|
||||
newest = n.At
|
||||
}
|
||||
}
|
||||
return newest.IsZero() || now.Sub(newest) > 26*time.Hour
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
package main
|
||||
|
||||
// The machine's backups over a fake restic and fake stores (novox/hq ADR 0214, to-be 43), and — where
|
||||
// restic is installed — over the real one, on throwaway directories.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
const composed = "# What the modules on this machine back up, composed by the mesh. Do not edit.\n" +
|
||||
"# postgres\nrun docker exec -u postgres postgres sh -c 'pg-dump-all'\npath /var/lib/mesh-store/dumps\n" +
|
||||
"# mailu\npath /var/lib/mailu/data-mail\npath /var/lib/mailu/data-dkim\n"
|
||||
|
||||
func placed(t *testing.T, declared string) Where {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
must(t, os.WriteFile(filepath.Join(dir, "backups.conf"), []byte(declared), 0o600))
|
||||
must(t, os.WriteFile(filepath.Join(dir, "pw"), []byte("secret\n"), 0o600))
|
||||
return Where{Declared: filepath.Join(dir, "backups.conf"), Repository: filepath.Join(dir, "repo"),
|
||||
PasswordFile: filepath.Join(dir, "pw"), State: dir}
|
||||
}
|
||||
|
||||
func must(t *testing.T, err error) {
|
||||
t.Helper()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func quiet(string, ...any) {}
|
||||
|
||||
func TestTheComposedFileIsReadIntoEachModulesRunsAndPaths(t *testing.T) {
|
||||
got, err := parseDeclared(composed)
|
||||
must(t, err)
|
||||
want := []Declared{
|
||||
{Module: "postgres", Runs: []string{"docker exec -u postgres postgres sh -c 'pg-dump-all'"}, Paths: []string{"/var/lib/mesh-store/dumps"}},
|
||||
{Module: "mailu", Paths: []string{"/var/lib/mailu/data-mail", "/var/lib/mailu/data-dkim"}},
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("read %#v, want %#v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestALineThisHolderDoesNotReadIsRefusedNamingTheModule(t *testing.T) {
|
||||
for _, bad := range []string{"# pg\ncopy /x\n", "# pg\npath relative/dir\n"} {
|
||||
if _, err := parseDeclared(bad); err == nil || !strings.Contains(err.Error(), "pg contributes a backup line") {
|
||||
t.Errorf("%q: %v", bad, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResticAndTheDumpsRunThroughSudoWhereTheAccountIsNotRoot(t *testing.T) {
|
||||
if p, a := escalated(1000, "restic", []string{"snapshots"}); p != "sudo" || !reflect.DeepEqual(a, []string{"-n", "restic", "snapshots"}) {
|
||||
t.Errorf("as an account: %s %v", p, a)
|
||||
}
|
||||
if p, a := escalated(0, "restic", []string{"snapshots"}); p != "restic" || !reflect.DeepEqual(a, []string{"snapshots"}) {
|
||||
t.Errorf("as root: %s %v", p, a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestANightDumpsBeforeEachSnapshotAndOneFailingModuleFailsOnlyItself(t *testing.T) {
|
||||
where := placed(t, "# pg\nrun dump-it\npath /\n# broken\nrun fail-it\npath /\n# mail\npath /\n")
|
||||
var calls []string
|
||||
run := func(_ context.Context, name string, args ...string) (string, error) {
|
||||
line := name + " " + strings.Join(args, " ")
|
||||
if name == "restic" {
|
||||
line = "restic " + strings.Join(args[5:], " ")
|
||||
}
|
||||
calls = append(calls, line)
|
||||
switch {
|
||||
case line == "sh -c fail-it":
|
||||
return "", errors.New("the dump failed")
|
||||
case strings.HasPrefix(line, "restic backup"):
|
||||
return "{\"message_type\":\"status\"}\n{\"message_type\":\"summary\",\"snapshot_id\":\"abcdef0123456789\"}\n", nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
b := &Backups{Where: where, Run: run, Now: func() time.Time { return time.Date(2026, 10, 6, 3, 0, 0, 0, time.UTC) }, Say: quiet}
|
||||
outcome, err := b.BackUp(context.Background(), "")
|
||||
must(t, err)
|
||||
if !outcome["pg"].OK || outcome["pg"].Snapshot != "abcdef01" {
|
||||
t.Errorf("pg: %+v", outcome["pg"])
|
||||
}
|
||||
if outcome["broken"].OK || !strings.Contains(outcome["broken"].Error, "the dump failed") {
|
||||
t.Errorf("broken: %+v", outcome["broken"])
|
||||
}
|
||||
if !outcome["mail"].OK {
|
||||
t.Errorf("mail failed with broken: %+v", outcome["mail"])
|
||||
}
|
||||
want := []string{
|
||||
"restic cat config",
|
||||
"sh -c dump-it",
|
||||
"restic backup --json --tag module=pg /",
|
||||
"sh -c fail-it",
|
||||
"restic backup --json --tag module=mail /",
|
||||
"restic forget --prune --group-by host,tags --keep-daily 14 --keep-weekly 8 --keep-monthly 6",
|
||||
}
|
||||
if !reflect.DeepEqual(calls, want) {
|
||||
t.Errorf("ran\n%s\nwant\n%s", strings.Join(calls, "\n"), strings.Join(want, "\n"))
|
||||
}
|
||||
// Recorded, so `backed-up` and the missed-night check read it.
|
||||
var nights map[string]Night
|
||||
raw, err := os.ReadFile(filepath.Join(where.State, "nights.json"))
|
||||
must(t, err)
|
||||
must(t, json.Unmarshal(raw, &nights))
|
||||
if nights["broken"].OK || !nights["mail"].OK {
|
||||
t.Errorf("recorded %+v", nights)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARepositoryThatWillNotOpenIsNeverReplaced(t *testing.T) {
|
||||
var calls []string
|
||||
run := func(_ context.Context, _ string, args ...string) (string, error) {
|
||||
calls = append(calls, strings.Join(args[5:], " "))
|
||||
if args[5] == "cat" {
|
||||
return "", errors.New("Fatal: wrong password or no key found")
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
b := &Backups{Where: placed(t, "# pg\npath /\n"), Run: run, Now: time.Now, Say: quiet}
|
||||
if _, err := b.BackUp(context.Background(), ""); err == nil || !strings.Contains(err.Error(), "cannot be opened, and is left as it is") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
for _, c := range calls {
|
||||
if c == "init" {
|
||||
t.Fatal("it made a new repository over one it could not open")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestADeclaredDirectoryThatDoesNotExistFailsThatModulesNight(t *testing.T) {
|
||||
b := &Backups{Where: placed(t, "# pg\npath /nowhere/at/all\n"), Run: func(context.Context, string, ...string) (string, error) { return "", nil },
|
||||
Now: time.Now, Say: quiet}
|
||||
outcome, err := b.BackUp(context.Background(), "")
|
||||
must(t, err)
|
||||
if outcome["pg"].OK || !strings.Contains(outcome["pg"].Error, "/nowhere/at/all does not exist") {
|
||||
t.Fatalf("pg: %+v", outcome["pg"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestANightIsDueAtTheHourAndAMissedOneIsNoticed(t *testing.T) {
|
||||
morning := time.Date(2026, 10, 6, 1, 30, 0, 0, time.Local)
|
||||
if got := nextNight(morning, 3); !got.Equal(time.Date(2026, 10, 6, 3, 0, 0, 0, time.Local)) {
|
||||
t.Errorf("from the morning: %v", got)
|
||||
}
|
||||
afternoon := time.Date(2026, 10, 6, 15, 0, 0, 0, time.Local)
|
||||
if got := nextNight(afternoon, 3); !got.Equal(time.Date(2026, 10, 7, 3, 0, 0, 0, time.Local)) {
|
||||
t.Errorf("from the afternoon: %v", got)
|
||||
}
|
||||
at := func(day int, ok bool) map[string]Night {
|
||||
return map[string]Night{"pg": {OK: ok, At: time.Date(2026, 10, day, 3, 5, 0, 0, time.Local)}}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
nights map[string]Night
|
||||
missed bool
|
||||
}{{map[string]Night{}, true}, {at(6, true), false}, {at(4, true), true}, {at(6, false), true}} {
|
||||
if got := missedANight(c.nights, afternoon); got != c.missed {
|
||||
t.Errorf("%+v: missed %v", c.nights, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The real thing, where restic is installed: a dump, a snapshot, a mistake, and a restore beside.
|
||||
func TestWithTheRealResticAMistakeIsUndoneBesideTheLiveData(t *testing.T) {
|
||||
if _, err := exec.LookPath("restic"); err != nil {
|
||||
t.Skip("restic is not installed")
|
||||
}
|
||||
root := t.TempDir()
|
||||
store, dumps := filepath.Join(root, "store"), filepath.Join(root, "dumps")
|
||||
must(t, os.Mkdir(store, 0o700))
|
||||
must(t, os.Mkdir(dumps, 0o700))
|
||||
must(t, os.WriteFile(filepath.Join(store, "mailbox"), []byte("the only copy of a letter\n"), 0o600))
|
||||
where := placed(t, "# mail\npath "+store+"\n# pg\nrun echo 'every row' > "+dumps+"/all.dump\npath "+dumps+"\n")
|
||||
// As whoever runs the test, against its own repository: no sudo.
|
||||
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
||||
out, err := exec.CommandContext(ctx, name, args...).Output()
|
||||
if ee, ok := err.(*exec.ExitError); ok {
|
||||
return string(out), errors.New(string(ee.Stderr))
|
||||
}
|
||||
return string(out), err
|
||||
}
|
||||
b := &Backups{Where: where, Run: run, Now: func() time.Time { return time.Date(2026, 10, 6, 3, 0, 0, 0, time.UTC) }, Say: quiet}
|
||||
ctx := context.Background()
|
||||
|
||||
night, err := b.BackUp(ctx, "")
|
||||
must(t, err)
|
||||
if !night["mail"].OK || !night["pg"].OK {
|
||||
t.Fatalf("the night: %+v", night)
|
||||
}
|
||||
if raw, _ := os.ReadFile(filepath.Join(dumps, "all.dump")); string(raw) != "every row\n" {
|
||||
t.Fatalf("the dump: %q", raw)
|
||||
}
|
||||
|
||||
// The mistake.
|
||||
must(t, os.Remove(filepath.Join(store, "mailbox")))
|
||||
|
||||
listed, err := b.BackedUp(ctx, "")
|
||||
must(t, err)
|
||||
if len(listed) != 2 || listed[0].RestorePoints != 1 || listed[1].RestorePoints != 1 {
|
||||
t.Fatalf("listed %+v", listed)
|
||||
}
|
||||
|
||||
restored, err := b.Restore(ctx, "mail", "", "")
|
||||
must(t, err)
|
||||
if len(restored.Restored) != 1 || !strings.HasSuffix(restored.Restored[0], "store.restored-20261006-030000") {
|
||||
t.Fatalf("restored %+v", restored)
|
||||
}
|
||||
if raw, _ := os.ReadFile(filepath.Join(restored.Restored[0], "mailbox")); string(raw) != "the only copy of a letter\n" {
|
||||
t.Fatalf("the restored letter: %q", raw)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(store, "mailbox")); err == nil {
|
||||
t.Fatal("the restore wrote into the live directory")
|
||||
}
|
||||
|
||||
// Never over anything: the same restore again finds its target taken.
|
||||
if _, err := b.Restore(ctx, "mail", "", ""); err == nil || !strings.Contains(err.Error(), "nothing is restored over anything") {
|
||||
t.Fatalf("a second restore: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
// restic-backups (novox/hq ADR 0214, to-be 43): the machine's backups. One binary, launched by the
|
||||
// machine's tool runtime and speaking MCP to it over stdio through the Go SDK (ADR 0193, ADR 0198). It
|
||||
// serves the node-backup seat's three verbs — what is backed up, take one now, restore beside the
|
||||
// live data — and, beside them, runs the night that happens without anyone asking.
|
||||
//
|
||||
// stdout is the MCP channel; everything this module says, it says on stderr.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// Seat is the role this module holds.
|
||||
const Seat = "node-backup"
|
||||
|
||||
func say(format string, args ...any) {
|
||||
fmt.Fprintf(os.Stderr, "[restic] "+format+"\n", args...)
|
||||
}
|
||||
|
||||
func main() {
|
||||
where, err := whereFromEnv()
|
||||
if err != nil {
|
||||
say("%v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
b := &Backups{Where: where, Run: execRunner, Now: time.Now, Say: say}
|
||||
go nights(b)
|
||||
if err := stdio.Serve("", tools(b)); err != nil {
|
||||
say("%v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// nights runs at the hour, every module; and at start, if a night was missed — the machine was off
|
||||
// or this module was not running at the hour — one soon rather than a day later. Not at once: a
|
||||
// machine just started has its stores still coming up.
|
||||
func nights(b *Backups) {
|
||||
hour := 3
|
||||
if h, err := strconv.Atoi(os.Getenv("MESH_BACKUP_HOUR")); err == nil && h >= 0 && h < 24 {
|
||||
hour = h
|
||||
}
|
||||
if missedANight(b.Nights(), time.Now()) {
|
||||
time.Sleep(10 * time.Minute)
|
||||
night(b)
|
||||
}
|
||||
for {
|
||||
time.Sleep(time.Until(nextNight(time.Now(), hour)))
|
||||
night(b)
|
||||
}
|
||||
}
|
||||
|
||||
func night(b *Backups) {
|
||||
ctx := context.Background()
|
||||
outcome, err := b.BackUp(ctx, "")
|
||||
if err != nil {
|
||||
say("the night did not run: %v", err)
|
||||
return
|
||||
}
|
||||
var failed []string
|
||||
for module, n := range outcome {
|
||||
if !n.OK {
|
||||
failed = append(failed, module)
|
||||
}
|
||||
}
|
||||
if len(failed) > 0 {
|
||||
say("the night left %s without a backup", strings.Join(failed, ", "))
|
||||
}
|
||||
// Sundays, the repository's own integrity with a sample of the data read back.
|
||||
if time.Now().Weekday() == time.Sunday {
|
||||
if err := b.Check(ctx); err != nil {
|
||||
say("the repository does NOT check out: %v", err)
|
||||
} else {
|
||||
say("the repository checks out")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- the seat's verbs --------------------------------------------------------------------------
|
||||
|
||||
func str(description string) map[string]any {
|
||||
return map[string]any{"type": "string", "description": description}
|
||||
}
|
||||
|
||||
func arg(a map[string]any, k string) string {
|
||||
v, _ := a[k].(string)
|
||||
return strings.TrimSpace(v)
|
||||
}
|
||||
|
||||
// verb is one of the seat's verbs: listed as `<seat>.<verb>`, so the runtime serves it on the seat's
|
||||
// subject.
|
||||
func verb(name, description string, input map[string]any, run func(a map[string]any) (any, error)) stdio.Tool {
|
||||
return stdio.Tool{Name: Seat + "." + name, Description: description, Input: input, Run: run}
|
||||
}
|
||||
|
||||
func tools(b *Backups) []stdio.Tool {
|
||||
return []stdio.Tool{
|
||||
verb("backed-up", "What this machine backs up: each module, what it declared, its last good night, how many restore points are kept.",
|
||||
map[string]any{"module": str("one module (optional)")},
|
||||
func(a map[string]any) (any, error) { return b.BackedUp(context.Background(), arg(a, "module")) }),
|
||||
verb("now", "Take a backup now, of one module or of every module on this machine — before a migration, a retirement or anything else that could go wrong. Answers when it has started; `backed-up` says how it went.",
|
||||
map[string]any{"module": str("one module (optional)")},
|
||||
func(a map[string]any) (any, error) {
|
||||
only := arg(a, "module")
|
||||
// A night of a large store outlasts any call; it is started, and its outcome recorded.
|
||||
go func() {
|
||||
if _, err := b.BackUp(context.Background(), only); err != nil {
|
||||
say("a backup asked for now failed: %v", err)
|
||||
}
|
||||
}()
|
||||
started := only
|
||||
if started == "" {
|
||||
started = "every module on this machine"
|
||||
}
|
||||
return map[string]any{"started": started, "follow": Seat + ".backed-up"}, nil
|
||||
}),
|
||||
verb("restore", "Restore one module's data from a restore point BESIDE the live data, never over it: each directory as <path>.restored-<date>. Swapping it in is a person's act.",
|
||||
map[string]any{
|
||||
"module": str("the module"),
|
||||
"snapshot": str("the restore point (the newest when omitted)"),
|
||||
"path": str("one of the module's directories (all of them when omitted)"),
|
||||
},
|
||||
func(a map[string]any) (any, error) {
|
||||
module := arg(a, "module")
|
||||
if module == "" {
|
||||
return nil, fmt.Errorf("module is required")
|
||||
}
|
||||
return b.Restore(context.Background(), module, arg(a, "snapshot"), arg(a, "path"))
|
||||
}),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
module restic
|
||||
|
||||
go 1.25.0
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.7
|
||||
@@ -0,0 +1,2 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
||||
@@ -45,12 +45,12 @@
|
||||
{
|
||||
"name": "tools",
|
||||
"kind": "bundle",
|
||||
"language": "typescript",
|
||||
"entrypoints": [
|
||||
"tools/index.js"
|
||||
],
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/restic-backups",
|
||||
"binary": "restic-backups",
|
||||
"loads": [
|
||||
"tools/index.js"
|
||||
"restic-backups"
|
||||
],
|
||||
"env": {
|
||||
"MESH_BACKUP_DECLARED": "${dir:state}/backups.conf",
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
{
|
||||
"name": "@novox/module-restic",
|
||||
"version": "0.1.0",
|
||||
"description": "restic \u2014 the machine's backups: holds the node-backup seat, takes a nightly restore point of what every module on the machine declares, keeps 14 daily, 8 weekly and 6 monthly, and restores beside the live data (novox/hq ADR 0214, to-be 43).",
|
||||
"type": "module",
|
||||
"private": true,
|
||||
"dependencies": {
|
||||
"@novox/mesh-sdk": "^0.1.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.0.0",
|
||||
"typescript": "^5.6.0"
|
||||
},
|
||||
"scripts": {
|
||||
"build": "tsc client.ts tools/index.ts --module NodeNext --moduleResolution NodeNext --target ES2022 --rootDir . --outDir dist",
|
||||
"test": "node --test --experimental-strip-types 'test/*.test.ts'"
|
||||
}
|
||||
}
|
||||
@@ -1,142 +0,0 @@
|
||||
// The machine's backups over a fake restic and fake stores (novox/hq ADR 0214, to-be 43), and — where
|
||||
// restic is installed — over the real one, on throwaway directories.
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { Backups, escalated, missedANight, nextNight, parseDeclared, type Runner } from "../client.ts";
|
||||
|
||||
const COMPOSED =
|
||||
"# What the modules on this machine back up, composed by the mesh. Do not edit.\n" +
|
||||
"# postgres\nrun docker exec -u postgres postgres sh -c 'pg-dump-all'\npath /var/lib/mesh-store/dumps\n" +
|
||||
"# mailu\npath /var/lib/mailu/data-mail\npath /var/lib/mailu/data-dkim\n";
|
||||
|
||||
function place(declared: string) {
|
||||
const dir = mkdtempSync(join(tmpdir(), "restic-test-"));
|
||||
writeFileSync(join(dir, "backups.conf"), declared);
|
||||
writeFileSync(join(dir, "pw"), "secret\n");
|
||||
return { declared: join(dir, "backups.conf"), repository: join(dir, "repo"), passwordFile: join(dir, "pw"), state: dir };
|
||||
}
|
||||
|
||||
test("the composed file is read into each module's runs and paths, the header comment being nothing", () => {
|
||||
assert.deepEqual(parseDeclared(COMPOSED), [
|
||||
{ module: "postgres", runs: ["docker exec -u postgres postgres sh -c 'pg-dump-all'"], paths: ["/var/lib/mesh-store/dumps"] },
|
||||
{ module: "mailu", runs: [], paths: ["/var/lib/mailu/data-mail", "/var/lib/mailu/data-dkim"] },
|
||||
]);
|
||||
});
|
||||
|
||||
test("a line this holder does not read is refused, naming the module, rather than skipped", () => {
|
||||
assert.throws(() => parseDeclared("# pg\ncopy /x\n"), /pg contributes a backup line this holder does not read: copy \/x/);
|
||||
assert.throws(() => parseDeclared("# pg\npath relative/dir\n"), /pg contributes/);
|
||||
});
|
||||
|
||||
test("restic and the dumps run through sudo where the account is not root", () => {
|
||||
assert.deepEqual(escalated("restic", ["snapshots"], 1000), ["sudo", ["-n", "restic", "snapshots"]]);
|
||||
assert.deepEqual(escalated("restic", ["snapshots"], 0), ["restic", ["snapshots"]]);
|
||||
});
|
||||
|
||||
test("a night runs each module's dump before its snapshot, and one failing module fails only itself", async () => {
|
||||
const where = place("# pg\nrun dump-it\npath /\n# broken\nrun fail-it\npath /\n# mail\npath /\n");
|
||||
const calls: string[] = [];
|
||||
const run: Runner = async (cmd, args) => {
|
||||
const line = cmd === "restic" ? `restic ${args.slice(5).join(" ")}` : `${cmd} ${args.join(" ")}`;
|
||||
calls.push(line);
|
||||
if (line === "sh -c fail-it") throw new Error("the dump failed");
|
||||
if (line.startsWith("restic backup")) return '{"message_type":"status"}\n{"message_type":"summary","snapshot_id":"abcdef0123456789"}\n';
|
||||
return "";
|
||||
};
|
||||
const outcome = await new Backups(where, run, () => new Date("2026-10-06T03:00:00Z"), () => {}).backUp();
|
||||
assert.equal(outcome.pg.ok, true);
|
||||
assert.equal(outcome.pg.snapshot, "abcdef01");
|
||||
assert.equal(outcome.broken.ok, false);
|
||||
assert.match(outcome.broken.error ?? "", /the dump failed/);
|
||||
assert.equal(outcome.mail.ok, true);
|
||||
assert.deepEqual(calls, [
|
||||
"restic cat config",
|
||||
"sh -c dump-it",
|
||||
"restic backup --json --tag module=pg /",
|
||||
"sh -c fail-it",
|
||||
"restic backup --json --tag module=mail /",
|
||||
"restic forget --prune --group-by host,tags --keep-daily 14 --keep-weekly 8 --keep-monthly 6",
|
||||
]);
|
||||
// Recorded, so `backed-up` and the missed-night check read it.
|
||||
const nights = JSON.parse(readFileSync(join(where.state, "nights.json"), "utf8"));
|
||||
assert.equal(nights.broken.ok, false);
|
||||
assert.equal(nights.mail.ok, true);
|
||||
});
|
||||
|
||||
test("a repository that exists and will not open is never replaced", async () => {
|
||||
const where = place("# pg\npath /\n");
|
||||
const calls: string[] = [];
|
||||
const run: Runner = async (cmd, args) => {
|
||||
calls.push(args.slice(5).join(" "));
|
||||
if (args.includes("cat")) throw new Error("Fatal: wrong password or no key found");
|
||||
return "";
|
||||
};
|
||||
await assert.rejects(new Backups(where, run, undefined, () => {}).backUp(), /cannot be opened, and is left as it is/);
|
||||
assert.ok(!calls.includes("init"), "it made a new repository over one it could not open");
|
||||
});
|
||||
|
||||
test("a declared directory that does not exist fails that module's night", async () => {
|
||||
const where = place("# pg\npath /nowhere/at/all\n");
|
||||
const run: Runner = async () => "";
|
||||
const outcome = await new Backups(where, run, undefined, () => {}).backUp();
|
||||
assert.equal(outcome.pg.ok, false);
|
||||
assert.match(outcome.pg.error ?? "", /\/nowhere\/at\/all does not exist/);
|
||||
});
|
||||
|
||||
test("a night is due at the hour today while it is ahead, else tomorrow; a missed one is noticed", () => {
|
||||
const morning = new Date(2026, 9, 6, 1, 30);
|
||||
assert.equal(nextNight(morning, 3).getTime(), new Date(2026, 9, 6, 3, 0).getTime());
|
||||
const afternoon = new Date(2026, 9, 6, 15, 0);
|
||||
assert.equal(nextNight(afternoon, 3).getTime(), new Date(2026, 9, 7, 3, 0).getTime());
|
||||
assert.equal(missedANight({}, afternoon), true);
|
||||
assert.equal(missedANight({ pg: { ok: true, at: new Date(2026, 9, 6, 3, 5).toISOString() } }, afternoon), false);
|
||||
assert.equal(missedANight({ pg: { ok: true, at: new Date(2026, 9, 4, 3, 5).toISOString() } }, afternoon), true);
|
||||
assert.equal(missedANight({ pg: { ok: false, at: new Date(2026, 9, 6, 3, 5).toISOString() } }, afternoon), true);
|
||||
});
|
||||
|
||||
// The real thing, where restic is installed: a dump, a snapshot, a mistake, and a restore beside.
|
||||
const hasRestic = (() => {
|
||||
try {
|
||||
execFileSync("restic", ["version"], { stdio: "ignore" });
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
})();
|
||||
|
||||
test("with the real restic: a mistake is undone by a restore beside the live data", { skip: !hasRestic && "restic is not installed" }, async () => {
|
||||
const root = mkdtempSync(join(tmpdir(), "restic-real-"));
|
||||
const store = join(root, "store");
|
||||
const dumps = join(root, "dumps");
|
||||
mkdirSync(store);
|
||||
mkdirSync(dumps);
|
||||
writeFileSync(join(store, "mailbox"), "the only copy of a letter\n");
|
||||
const where = place(`# mail\npath ${store}\n# pg\nrun echo 'every row' > ${dumps}/all.dump\npath ${dumps}\n`);
|
||||
const backups = new Backups(where, undefined, () => new Date("2026-10-06T03:00:00Z"), () => {});
|
||||
// escalated() would sudo; the test runs as whoever it is, against its own repository.
|
||||
(backups as unknown as { run: Runner }).run = async (cmd, args) => execFileSync(cmd, args, { encoding: "utf8" });
|
||||
|
||||
const night = await backups.backUp();
|
||||
assert.equal(night.mail.ok, true, night.mail.error);
|
||||
assert.equal(night.pg.ok, true, night.pg.error);
|
||||
assert.equal(readFileSync(join(dumps, "all.dump"), "utf8"), "every row\n");
|
||||
|
||||
// The mistake.
|
||||
rmSync(join(store, "mailbox"));
|
||||
|
||||
const listed = await backups.backedUp();
|
||||
assert.deepEqual(listed.map((m) => [m.module, m.restorePoints]), [["mail", 1], ["pg", 1]]);
|
||||
|
||||
const restored = await backups.restore("mail");
|
||||
assert.equal(restored.restored.length, 1);
|
||||
assert.match(restored.restored[0], /store\.restored-20261006-030000$/);
|
||||
assert.equal(readFileSync(join(restored.restored[0], "mailbox"), "utf8"), "the only copy of a letter\n");
|
||||
assert.ok(!existsSync(join(store, "mailbox")), "the restore wrote into the live directory");
|
||||
|
||||
// Never over anything: the same restore again finds its target taken.
|
||||
await assert.rejects(backups.restore("mail"), /already exists; nothing is restored over anything/);
|
||||
});
|
||||
@@ -1,78 +0,0 @@
|
||||
// The machine's backups: the node-backup seat's three verbs — what is backed up, take one now,
|
||||
// restore beside the live data — and the night that runs without anyone asking (novox/hq ADR 0214,
|
||||
// to-be 43). What is backed up is composed by the mesh from the modules the machine runs; this code
|
||||
// only runs it.
|
||||
|
||||
import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools";
|
||||
import { Backups, missedANight, nextNight, whereFromEnv } from "../client.js";
|
||||
|
||||
export function getSeatVerbs(backups: Backups): ToolDefinition[] {
|
||||
return [
|
||||
{
|
||||
name: "backed-up",
|
||||
description:
|
||||
"What this machine backs up: each module, what it declared, its last good night, how many restore points are kept.",
|
||||
input: { module: { type: "string", description: "one module (optional)" } },
|
||||
run: async (args) => backups.backedUp(args.module ? String(args.module) : undefined),
|
||||
},
|
||||
{
|
||||
name: "now",
|
||||
description:
|
||||
"Take a backup now, of one module or of every module on this machine — before a migration, a retirement or anything else that could go wrong. Answers when it has started; `backed-up` says how it went.",
|
||||
input: { module: { type: "string", description: "one module (optional)" } },
|
||||
run: async (args) => {
|
||||
const only = args.module ? String(args.module) : undefined;
|
||||
// A night of a large store outlasts any call; it is started, and its outcome recorded.
|
||||
backups.backUp(only).catch((err) => console.error(`[restic] a backup asked for now failed: ${(err as Error).message}`));
|
||||
return { started: only ?? "every module on this machine", follow: "backed-up" };
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "restore",
|
||||
description:
|
||||
"Restore one module's data from a restore point BESIDE the live data, never over it: each directory as <path>.restored-<date>. Swapping it in is a person's act.",
|
||||
input: {
|
||||
module: { type: "string", description: "the module" },
|
||||
snapshot: { type: "string", description: "the restore point (the newest when omitted)" },
|
||||
path: { type: "string", description: "one of the module's directories (all of them when omitted)" },
|
||||
},
|
||||
run: async (args) =>
|
||||
backups.restore(String(args.module ?? ""), args.snapshot ? String(args.snapshot) : undefined, args.path ? String(args.path) : undefined),
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
const backups = new Backups(whereFromEnv());
|
||||
registerModuleTools("node-backup", () => getSeatVerbs(backups));
|
||||
|
||||
// The night. At the hour, every module; and at start, if a night was missed — the machine was off
|
||||
// or this module was not running at the hour — one now rather than a day later.
|
||||
const HOUR = Number(process.env.MESH_BACKUP_HOUR ?? "3");
|
||||
|
||||
async function night(): Promise<void> {
|
||||
try {
|
||||
const outcome = await backups.backUp();
|
||||
const failed = Object.entries(outcome).filter(([, n]) => !n.ok).map(([m]) => m);
|
||||
if (failed.length > 0) console.error(`[restic] the night left ${failed.join(", ")} without a backup`);
|
||||
// Sundays, the repository's own integrity with a sample of the data read back.
|
||||
if (new Date().getDay() === 0) {
|
||||
await backups.check().then(
|
||||
() => console.error("[restic] the repository checks out"),
|
||||
(err) => console.error(`[restic] the repository does NOT check out: ${(err as Error).message}`),
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`[restic] the night did not run: ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
function schedule(): void {
|
||||
const at = nextNight(new Date(), HOUR);
|
||||
setTimeout(() => void night().finally(schedule), at.getTime() - Date.now());
|
||||
}
|
||||
|
||||
if (missedANight(backups.nights(), new Date())) {
|
||||
// Not at once: a machine just started has its stores still coming up.
|
||||
setTimeout(() => void night(), 10 * 60 * 1000);
|
||||
}
|
||||
schedule();
|
||||
@@ -1,15 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"noEmit": true
|
||||
},
|
||||
"include": [
|
||||
"client.ts",
|
||||
"tools/index.ts"
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user