diff --git a/modules/gitea/module.json b/modules/gitea/module.json index c329469..16175fc 100644 --- a/modules/gitea/module.json +++ b/modules/gitea/module.json @@ -222,5 +222,12 @@ "failregex": "^.*Failed authentication attempt for .* from (?::\\d+)?\\s*$\n ^.*Invalid user .* from port \\d+\\s*$\n ^.*User \\S+ from not allowed because .*$", "jail": "backend = systemd\njournalmatch = CONTAINER_NAME=gitea\nport = http,https,222\nmaxretry = 3\nfindtime = 1d\nbantime = 1d" } + ], + "contributions": [ + { + "seat": "node-backup", + "kind": "backup", + "content": "path ${dir:data}\n" + } ] } diff --git a/modules/influxdb/module.json b/modules/influxdb/module.json index 0232510..fbc8a67 100644 --- a/modules/influxdb/module.json +++ b/modules/influxdb/module.json @@ -132,5 +132,12 @@ } } ] - } + }, + "contributions": [ + { + "seat": "node-backup", + "kind": "backup", + "content": "path ${dir:data}\npath ${dir:config}\n" + } + ] } diff --git a/modules/mailu/module.json b/modules/mailu/module.json index 4a1cd66..ac5d45c 100644 --- a/modules/mailu/module.json +++ b/modules/mailu/module.json @@ -558,5 +558,12 @@ "failregex": "^.*(?:imap|pop3|submission|managesieve)-login: .*\\(auth failed, \\d+ attempts(?: in \\d+ secs)?\\):.*rip=(?:,|$)", "jail": "backend = systemd\njournalmatch = CONTAINER_NAME=mailu-front\nport = smtp,submission,submissions,imap,imaps,pop3,pop3s\nmaxretry = 3\nfindtime = 1d\nbantime = 1d" } + ], + "contributions": [ + { + "seat": "node-backup", + "kind": "backup", + "content": "path ${dir:data-mail}\npath ${dir:data-dkim}\npath ${dir:data-data}\npath ${dir:data-dav}\npath ${dir:data-webmail}\n" + } ] } diff --git a/modules/mesh-vault/module.json b/modules/mesh-vault/module.json index 31c3092..f68e8ed 100644 --- a/modules/mesh-vault/module.json +++ b/modules/mesh-vault/module.json @@ -79,5 +79,12 @@ "name": "mesh-vault", "scope": "mesh" } + ], + "contributions": [ + { + "seat": "node-backup", + "kind": "backup", + "content": "path ${dir:state}\npath ${dir:ledger}\npath ${dir:root}\n" + } ] } diff --git a/modules/minio/module.json b/modules/minio/module.json index 6a0bc85..ac6b7aa 100644 --- a/modules/minio/module.json +++ b/modules/minio/module.json @@ -151,5 +151,12 @@ } } ] - } + }, + "contributions": [ + { + "seat": "node-backup", + "kind": "backup", + "content": "path ${dir:data}\n" + } + ] } diff --git a/modules/mongodb/module.json b/modules/mongodb/module.json index 1f70756..6454914 100644 --- a/modules/mongodb/module.json +++ b/modules/mongodb/module.json @@ -58,6 +58,11 @@ "type": "directory", "mode": "0700" }, + { + "id": "dumps", + "type": "directory", + "mode": "0700" + }, { "id": "net", "type": "network", @@ -113,5 +118,12 @@ } } ] - } + }, + "contributions": [ + { + "seat": "node-backup", + "kind": "backup", + "content": "run docker exec mongodb-server sh -c 'printf \"password: %s\\n\" \"$(cat /run/secrets/root)\" > /tmp/.backup.yaml && mongodump --quiet --config /tmp/.backup.yaml --username root --authenticationDatabase admin --archive; s=$?; rm -f /tmp/.backup.yaml; exit $s' > ${dir:dumps}/all.archive.partial && mv ${dir:dumps}/all.archive.partial ${dir:dumps}/all.archive\npath ${dir:dumps}\n" + } + ] } diff --git a/modules/mssql/module.json b/modules/mssql/module.json index 28802a7..5a44231 100644 --- a/modules/mssql/module.json +++ b/modules/mssql/module.json @@ -65,6 +65,13 @@ "mode": "0700", "owner": "10001:0" }, + { + "id": "dumps", + "type": "directory", + "path": "${dir:data}/backup", + "mode": "0700", + "owner": "10001:0" + }, { "id": "net", "type": "network", @@ -86,6 +93,13 @@ "${dir:data}:/var/opt/mssql" ], "secrets-in-environment": "the image documents only MSSQL_SA_PASSWORD, no _FILE and no configuration field; not convertible without a wrapper entrypoint" + }, + { + "id": "backup-sql", + "type": "file", + "path": "${dir:state}/backup.sql", + "mode": "0600", + "content": "SET NOCOUNT ON;\nDECLARE @n sysname, @s nvarchar(max);\nDECLARE c CURSOR LOCAL FAST_FORWARD FOR\n SELECT name FROM sys.databases WHERE database_id > 4 AND state = 0 AND source_database_id IS NULL;\nOPEN c;\nFETCH NEXT FROM c INTO @n;\nWHILE @@FETCH_STATUS = 0\nBEGIN\n SET @s = N'BACKUP DATABASE ' + QUOTENAME(@n) + N' TO DISK = N''/var/opt/mssql/backup/' + REPLACE(@n, N'''', N'''''') + N'.bak'' WITH INIT, COPY_ONLY, CHECKSUM';\n EXEC (@s);\n FETCH NEXT FROM c INTO @n;\nEND\nCLOSE c;\nDEALLOCATE c;\n" } ], "build": { @@ -112,5 +126,12 @@ } } ] - } + }, + "contributions": [ + { + "seat": "node-backup", + "kind": "backup", + "content": "run { cat ${dir:state}/sa.secret; echo; cat ${dir:state}/backup.sql; } | docker exec -i mssql sh -c 'read -r p; SQLCMDPASSWORD=\"$p\" exec /opt/mssql-tools18/bin/sqlcmd -C -b -S localhost -U sa -i /dev/stdin'\npath ${dir:dumps}\n" + } + ] } diff --git a/modules/nextcloud/module.json b/modules/nextcloud/module.json index 39fac38..fb177b4 100644 --- a/modules/nextcloud/module.json +++ b/modules/nextcloud/module.json @@ -117,5 +117,12 @@ } } ] - } + }, + "contributions": [ + { + "seat": "node-backup", + "kind": "backup", + "content": "path ${dir:html}\n" + } + ] } diff --git a/modules/postgres/module.json b/modules/postgres/module.json index f25d84d..dc1a256 100644 --- a/modules/postgres/module.json +++ b/modules/postgres/module.json @@ -74,6 +74,13 @@ "mode": "0700", "owner": "999:70" }, + { + "id": "dumps", + "type": "directory", + "path": "${dir:store-data}/dumps", + "mode": "0700", + "owner": "999:70" + }, { "id": "server", "type": "container", @@ -121,5 +128,12 @@ } } ] - } + }, + "contributions": [ + { + "seat": "node-backup", + "kind": "backup", + "content": "run docker exec -u postgres postgres sh -c 'cd /var/lib/postgresql/data/dumps && for db in $(psql -Atc \"select datname from pg_database where oid >= 16384 order by 1\"); do pg_dump -Fc -f \"$db.dump.partial\" \"$db\" && mv \"$db.dump.partial\" \"$db.dump\" || exit 1; done'\npath ${dir:dumps}\n" + } + ] } diff --git a/modules/restic/cmd/restic-backups/backups.go b/modules/restic/cmd/restic-backups/backups.go new file mode 100644 index 0000000..a65869e --- /dev/null +++ b/modules/restic/cmd/restic-backups/backups.go @@ -0,0 +1,472 @@ +// The machine's backups (novox/hq ADR 0214, to-be 43). +// +// The mesh composes what to back up: every module on the machine contributes `backup` lines to the +// node-backup seat, and the mesh writes them, each module's under a `# ` line and with its +// directories already filled, into one file this module reads. Two kinds of line: +// +// run run as root before the module's snapshot — a consistent dump of a store +// path a directory the module's snapshot keeps +// +// Each module gets one snapshot a night, tagged with its name, so a module is listed, kept and +// restored on its own. Everything lands in one repository on the machine — deduplicated, so every +// night is a complete restore point and only what changed costs space — and is thinned to 14 daily, +// 8 weekly and 6 monthly. Against mistakes, not disasters: nothing leaves the machine. +// +// Root's: the dumps read every store and the repository holds every module's data, and the runtime +// launching this binary runs as the operator's account, so restic and the run lines go through sudo +// without a prompt where the account is not root (ADR 0175 §4). +package main + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "slices" + "strings" + "sync" + "time" +) + +// Runner runs one command and answers what it printed, so the backups can be tested without restic +// or a store. +type Runner func(ctx context.Context, name string, args ...string) (string, error) + +// escalated is the command as it is run: as given when this process is root, else through sudo +// without a prompt. +func escalated(uid int, name string, args []string) (string, []string) { + if uid == 0 { + return name, args + } + return "sudo", append([]string{"-n", name}, args...) +} + +// execRunner runs it for real. A night's dump of a large store takes a while; six hours is a dump +// that will not finish. +func execRunner(ctx context.Context, name string, args ...string) (string, error) { + ctx, cancel := context.WithTimeout(ctx, 6*time.Hour) + defer cancel() + program, argv := escalated(os.Getuid(), name, args) + var stdout, stderr bytes.Buffer + cmd := exec.CommandContext(ctx, program, argv...) + cmd.Stdout, cmd.Stderr = &stdout, &stderr + if err := cmd.Run(); err != nil { + said := strings.TrimSpace(stderr.String()) + if said == "" { + said = strings.TrimSpace(stdout.String()) + } + if program == "sudo" && strings.HasPrefix(said, "sudo:") { + return "", fmt.Errorf("%s needs root and the runtime's account may not run it without a prompt: %s", name, said) + } + lines := strings.Split(said, "\n") + if len(lines) > 3 { + lines = lines[len(lines)-3:] + } + if said == "" { + return "", fmt.Errorf("%s: %w", name, err) + } + return "", errors.New(strings.Join(lines, " / ")) + } + return stdout.String(), nil +} + +// Declared is what one module declared. +type Declared struct { + Module string `json:"module"` + Runs []string `json:"runs"` + Paths []string `json:"paths"` +} + +var moduleHeader = regexp.MustCompile(`^#\s*([a-z0-9][a-z0-9-]*)$`) + +// parseDeclared reads the composed file into each module's declaration, in the order the mesh wrote +// them. A line before any module, a comment that is not a module's name, or a blank, is nothing; a +// line this holder does not read is refused, naming the module, rather than skipped. +func parseDeclared(text string) ([]Declared, error) { + var out []Declared + current := -1 + for _, raw := range strings.Split(text, "\n") { + line := strings.TrimSpace(raw) + if line == "" { + continue + } + if m := moduleHeader.FindStringSubmatch(line); m != nil { + out = append(out, Declared{Module: m[1]}) + current = len(out) - 1 + continue + } + if strings.HasPrefix(line, "#") || current < 0 { + continue + } + kind, value, _ := strings.Cut(line, " ") + value = strings.TrimSpace(value) + d := &out[current] + switch { + case kind == "run" && value != "": + d.Runs = append(d.Runs, value) + case kind == "path" && strings.HasPrefix(value, "/") && !strings.ContainsAny(value, " \t"): + d.Paths = append(d.Paths, value) + default: + return nil, fmt.Errorf("%s contributes a backup line this holder does not read: %s", d.Module, line) + } + } + kept := out[:0] + for _, d := range out { + if len(d.Runs) > 0 || len(d.Paths) > 0 { + kept = append(kept, d) + } + } + return kept, nil +} + +// Snapshot is one restore point, as restic lists it. +type Snapshot struct { + ID string `json:"id"` + ShortID string `json:"short_id"` + Time time.Time `json:"time"` + Paths []string `json:"paths"` + Tags []string `json:"tags"` + Hostname string `json:"hostname"` +} + +// Night is how one module's last night went. +type Night struct { + OK bool `json:"ok"` + At time.Time `json:"at"` + Snapshot string `json:"snapshot,omitempty"` + Error string `json:"error,omitempty"` +} + +// Keep is the rotation (novox/hq ADR 0214). +var Keep = struct{ Daily, Weekly, Monthly int }{14, 8, 6} + +func tagOf(module string) string { return "module=" + module } + +// Where is where the mesh put this module's things. +type Where struct { + Declared, Repository, PasswordFile, State string +} + +func whereFromEnv() (Where, error) { + var missing []string + get := func(k string) string { + v := os.Getenv(k) + if v == "" { + missing = append(missing, k) + } + return v + } + w := Where{ + Declared: get("MESH_BACKUP_DECLARED"), + Repository: get("MESH_BACKUP_REPOSITORY"), + PasswordFile: get("MESH_BACKUP_PASSWORD_FILE"), + State: get("MESH_BACKUP_STATE"), + } + if len(missing) > 0 { + return w, fmt.Errorf("%s not set; the mesh gives them to this module", strings.Join(missing, ", ")) + } + return w, nil +} + +// Backups is the machine's backups. One thing at a time: two nights, or a night and a restore, never +// share a dump. +type Backups struct { + Where Where + Run Runner + Now func() time.Time + Say func(format string, args ...any) + mu sync.Mutex +} + +func (b *Backups) restic(ctx context.Context, args ...string) (string, error) { + return b.Run(ctx, "restic", append([]string{"--repo", b.Where.Repository, "--password-file", b.Where.PasswordFile, "--no-cache"}, args...)...) +} + +// Declared is what the modules on this machine declared. +func (b *Backups) Declared() ([]Declared, error) { + raw, err := os.ReadFile(b.Where.Declared) + if err != nil { + return nil, err + } + return parseDeclared(string(raw)) +} + +func (b *Backups) nightsFile() string { return filepath.Join(b.Where.State, "nights.json") } + +// Nights is how each module's last night went. +func (b *Backups) Nights() map[string]Night { + nights := map[string]Night{} + if raw, err := os.ReadFile(b.nightsFile()); err == nil { + _ = json.Unmarshal(raw, &nights) + } + return nights +} + +func (b *Backups) record(module string, n Night) { + nights := b.Nights() + nights[module] = n + raw, _ := json.MarshalIndent(nights, "", " ") + if err := os.WriteFile(b.nightsFile(), append(raw, '\n'), 0o600); err != nil { + b.Say("recording %s's night failed: %v", module, err) + } +} + +var noRepository = regexp.MustCompile(`(?i)does not exist|unable to open config file|Is there a repository at the following location`) + +// ensureRepository makes the repository the first time. One that exists and cannot be opened is +// said, never replaced: replacing it would discard every restore point to fix a password. +func (b *Backups) ensureRepository(ctx context.Context) error { + _, err := b.restic(ctx, "cat", "config") + if err == nil { + return nil + } + if !noRepository.MatchString(err.Error()) { + return fmt.Errorf("the repository at %s cannot be opened, and is left as it is: %v", b.Where.Repository, err) + } + b.Say("no repository at %s; making one", b.Where.Repository) + _, err = b.restic(ctx, "init") + return err +} + +// one is one module's night: its run lines, then one snapshot of its paths. A failure is that +// module's alone. +func (b *Backups) one(ctx context.Context, d Declared) Night { + n := Night{At: b.Now().UTC()} + fail := func(err error) Night { + n.Error = err.Error() + b.Say("%s: NOT backed up: %s", d.Module, n.Error) + return n + } + for _, command := range d.Runs { + if _, err := b.Run(ctx, "sh", "-c", command); err != nil { + return fail(err) + } + } + if len(d.Paths) == 0 { + return fail(errors.New("it runs a dump and names no directory to keep it from")) + } + var missing []string + for _, p := range d.Paths { + if _, err := os.Stat(p); err != nil { + missing = append(missing, p) + } + } + if len(missing) > 0 { + return fail(fmt.Errorf("%s does not exist", strings.Join(missing, ", "))) + } + out, err := b.restic(ctx, append([]string{"backup", "--json", "--tag", tagOf(d.Module)}, d.Paths...)...) + if err != nil { + return fail(err) + } + scanner := bufio.NewScanner(strings.NewReader(out)) + scanner.Buffer(make([]byte, 1024*1024), 16*1024*1024) + for scanner.Scan() { + var m struct { + MessageType string `json:"message_type"` + SnapshotID string `json:"snapshot_id"` + } + if json.Unmarshal(scanner.Bytes(), &m) == nil && m.MessageType == "summary" && len(m.SnapshotID) >= 8 { + n.Snapshot = m.SnapshotID[:8] + } + } + n.OK = true + b.Say("%s: backed up (%s)", d.Module, n.Snapshot) + return n +} + +// BackUp is a night: every module, or one, then the rotation. It answers each module's outcome. +func (b *Backups) BackUp(ctx context.Context, only string) (map[string]Night, error) { + b.mu.Lock() + defer b.mu.Unlock() + if err := b.ensureRepository(ctx); err != nil { + return nil, err + } + all, err := b.Declared() + if err != nil { + return nil, err + } + chosen := all + if only != "" { + chosen = nil + var names []string + for _, d := range all { + names = append(names, d.Module) + if d.Module == only { + chosen = append(chosen, d) + } + } + if len(chosen) == 0 { + return nil, fmt.Errorf("%s declares nothing to back up on this machine; it backs up %s", only, orNothing(names)) + } + } + outcome := map[string]Night{} + for _, d := range chosen { + outcome[d.Module] = b.one(ctx, d) + b.record(d.Module, outcome[d.Module]) + } + if _, err := b.restic(ctx, "forget", "--prune", "--group-by", "host,tags", + "--keep-daily", fmt.Sprint(Keep.Daily), "--keep-weekly", fmt.Sprint(Keep.Weekly), "--keep-monthly", fmt.Sprint(Keep.Monthly)); err != nil { + b.Say("thinning the restore points failed, and every one is kept: %v", err) + } + return outcome, nil +} + +func orNothing(names []string) string { + if len(names) == 0 { + return "nothing" + } + return strings.Join(names, ", ") +} + +// Snapshots is the restore points, of one module or all. +func (b *Backups) Snapshots(ctx context.Context, module string) ([]Snapshot, error) { + args := []string{"snapshots", "--json"} + if module != "" { + args = append(args, "--tag", tagOf(module)) + } + out, err := b.restic(ctx, args...) + if err != nil { + return nil, err + } + var snaps []Snapshot + if strings.TrimSpace(out) == "" { + return nil, nil + } + if err := json.Unmarshal([]byte(out), &snaps); err != nil { + return nil, fmt.Errorf("restic listed its snapshots in a form this holder does not read: %v", err) + } + return snaps, nil +} + +// ModuleBackups is what `backed-up` says about one module. +type ModuleBackups struct { + Module string `json:"module"` + Runs int `json:"runs"` + Paths []string `json:"paths"` + LastNight *Night `json:"lastNight"` + RestorePoints int `json:"restorePoints"` + Newest *Snapshot `json:"newest,omitempty"` +} + +// BackedUp is what is backed up here: each module, what it declared, its last night and its restore +// points. +func (b *Backups) BackedUp(ctx context.Context, module string) ([]ModuleBackups, error) { + declared, err := b.Declared() + if err != nil { + return nil, err + } + nights := b.Nights() + snaps, _ := b.Snapshots(ctx, module) + out := []ModuleBackups{} + for _, d := range declared { + if module != "" && d.Module != module { + continue + } + m := ModuleBackups{Module: d.Module, Runs: len(d.Runs), Paths: d.Paths} + if n, ok := nights[d.Module]; ok { + m.LastNight = &n + } + for _, s := range snaps { + if slices.Contains(s.Tags, tagOf(d.Module)) { + m.RestorePoints++ + newest := s + m.Newest = &newest + } + } + out = append(out, m) + } + return out, nil +} + +// Restored is what a restore put where. +type Restored struct { + Module string `json:"module"` + From Snapshot `json:"from"` + Restored []string `json:"restored"` + Live string `json:"live"` +} + +// Restore puts a module's data from a restore point BESIDE the live data: each directory as +// .restored-. A target that already exists is refused, never overwritten. +func (b *Backups) Restore(ctx context.Context, module, snapshot, path string) (*Restored, error) { + b.mu.Lock() + defer b.mu.Unlock() + mine, err := b.Snapshots(ctx, module) + if err != nil { + return nil, err + } + if len(mine) == 0 { + return nil, fmt.Errorf("%s has no restore point on this machine", module) + } + chosen := mine[len(mine)-1] + if snapshot != "" { + found := false + var listed []string + for _, s := range mine { + listed = append(listed, fmt.Sprintf("%s (%s)", s.ShortID, s.Time.Format(time.RFC3339))) + if s.ShortID == snapshot || strings.HasPrefix(s.ID, snapshot) { + chosen, found = s, true + } + } + if !found { + return nil, fmt.Errorf("%s has no restore point %s; it has %s", module, snapshot, strings.Join(listed, ", ")) + } + } + paths := chosen.Paths + if path != "" { + if !slices.Contains(chosen.Paths, path) { + return nil, fmt.Errorf("restore point %s of %s holds %s, not %s", chosen.ShortID, module, strings.Join(chosen.Paths, ", "), path) + } + paths = []string{path} + } + stamp := b.Now().UTC().Format("20060102-150405") + r := &Restored{Module: module, From: chosen, Live: "untouched — swapping it in is a person's act"} + for _, p := range paths { + target := p + ".restored-" + stamp + if _, err := os.Stat(target); err == nil { + return nil, fmt.Errorf("%s already exists; nothing is restored over anything", target) + } + if _, err := b.restic(ctx, "restore", chosen.ID+":"+p, "--target", target); err != nil { + return nil, err + } + r.Restored = append(r.Restored, target) + b.Say("%s: restored %s from %s to %s", module, p, chosen.ShortID, target) + } + return r, nil +} + +// Check is the weekly look at the repository's own integrity, with a sample of the data read back. +func (b *Backups) Check(ctx context.Context) error { + b.mu.Lock() + defer b.mu.Unlock() + _, err := b.restic(ctx, "check", "--read-data-subset", "5%") + return err +} + +// nextNight is when the next night is due: the given hour, local time, today while it is still +// ahead, else tomorrow. +func nextNight(now time.Time, hour int) time.Time { + next := time.Date(now.Year(), now.Month(), now.Day(), hour, 0, 0, 0, now.Location()) + if !next.After(now) { + next = next.AddDate(0, 0, 1) + } + return next +} + +// missedANight is whether a night was missed: the newest good night of any module is older than a +// day and a bit — the machine was off, or this module was not running, at the hour. +func missedANight(nights map[string]Night, now time.Time) bool { + var newest time.Time + for _, n := range nights { + if n.OK && n.At.After(newest) { + newest = n.At + } + } + return newest.IsZero() || now.Sub(newest) > 26*time.Hour +} diff --git a/modules/restic/cmd/restic-backups/backups_test.go b/modules/restic/cmd/restic-backups/backups_test.go new file mode 100644 index 0000000..cf85a03 --- /dev/null +++ b/modules/restic/cmd/restic-backups/backups_test.go @@ -0,0 +1,228 @@ +package main + +// The machine's backups over a fake restic and fake stores (novox/hq ADR 0214, to-be 43), and — where +// restic is installed — over the real one, on throwaway directories. + +import ( + "context" + "encoding/json" + "errors" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "testing" + "time" +) + +const composed = "# What the modules on this machine back up, composed by the mesh. Do not edit.\n" + + "# postgres\nrun docker exec -u postgres postgres sh -c 'pg-dump-all'\npath /var/lib/mesh-store/dumps\n" + + "# mailu\npath /var/lib/mailu/data-mail\npath /var/lib/mailu/data-dkim\n" + +func placed(t *testing.T, declared string) Where { + t.Helper() + dir := t.TempDir() + must(t, os.WriteFile(filepath.Join(dir, "backups.conf"), []byte(declared), 0o600)) + must(t, os.WriteFile(filepath.Join(dir, "pw"), []byte("secret\n"), 0o600)) + return Where{Declared: filepath.Join(dir, "backups.conf"), Repository: filepath.Join(dir, "repo"), + PasswordFile: filepath.Join(dir, "pw"), State: dir} +} + +func must(t *testing.T, err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } +} + +func quiet(string, ...any) {} + +func TestTheComposedFileIsReadIntoEachModulesRunsAndPaths(t *testing.T) { + got, err := parseDeclared(composed) + must(t, err) + want := []Declared{ + {Module: "postgres", Runs: []string{"docker exec -u postgres postgres sh -c 'pg-dump-all'"}, Paths: []string{"/var/lib/mesh-store/dumps"}}, + {Module: "mailu", Paths: []string{"/var/lib/mailu/data-mail", "/var/lib/mailu/data-dkim"}}, + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("read %#v, want %#v", got, want) + } +} + +func TestALineThisHolderDoesNotReadIsRefusedNamingTheModule(t *testing.T) { + for _, bad := range []string{"# pg\ncopy /x\n", "# pg\npath relative/dir\n"} { + if _, err := parseDeclared(bad); err == nil || !strings.Contains(err.Error(), "pg contributes a backup line") { + t.Errorf("%q: %v", bad, err) + } + } +} + +func TestResticAndTheDumpsRunThroughSudoWhereTheAccountIsNotRoot(t *testing.T) { + if p, a := escalated(1000, "restic", []string{"snapshots"}); p != "sudo" || !reflect.DeepEqual(a, []string{"-n", "restic", "snapshots"}) { + t.Errorf("as an account: %s %v", p, a) + } + if p, a := escalated(0, "restic", []string{"snapshots"}); p != "restic" || !reflect.DeepEqual(a, []string{"snapshots"}) { + t.Errorf("as root: %s %v", p, a) + } +} + +func TestANightDumpsBeforeEachSnapshotAndOneFailingModuleFailsOnlyItself(t *testing.T) { + where := placed(t, "# pg\nrun dump-it\npath /\n# broken\nrun fail-it\npath /\n# mail\npath /\n") + var calls []string + run := func(_ context.Context, name string, args ...string) (string, error) { + line := name + " " + strings.Join(args, " ") + if name == "restic" { + line = "restic " + strings.Join(args[5:], " ") + } + calls = append(calls, line) + switch { + case line == "sh -c fail-it": + return "", errors.New("the dump failed") + case strings.HasPrefix(line, "restic backup"): + return "{\"message_type\":\"status\"}\n{\"message_type\":\"summary\",\"snapshot_id\":\"abcdef0123456789\"}\n", nil + } + return "", nil + } + b := &Backups{Where: where, Run: run, Now: func() time.Time { return time.Date(2026, 10, 6, 3, 0, 0, 0, time.UTC) }, Say: quiet} + outcome, err := b.BackUp(context.Background(), "") + must(t, err) + if !outcome["pg"].OK || outcome["pg"].Snapshot != "abcdef01" { + t.Errorf("pg: %+v", outcome["pg"]) + } + if outcome["broken"].OK || !strings.Contains(outcome["broken"].Error, "the dump failed") { + t.Errorf("broken: %+v", outcome["broken"]) + } + if !outcome["mail"].OK { + t.Errorf("mail failed with broken: %+v", outcome["mail"]) + } + want := []string{ + "restic cat config", + "sh -c dump-it", + "restic backup --json --tag module=pg /", + "sh -c fail-it", + "restic backup --json --tag module=mail /", + "restic forget --prune --group-by host,tags --keep-daily 14 --keep-weekly 8 --keep-monthly 6", + } + if !reflect.DeepEqual(calls, want) { + t.Errorf("ran\n%s\nwant\n%s", strings.Join(calls, "\n"), strings.Join(want, "\n")) + } + // Recorded, so `backed-up` and the missed-night check read it. + var nights map[string]Night + raw, err := os.ReadFile(filepath.Join(where.State, "nights.json")) + must(t, err) + must(t, json.Unmarshal(raw, &nights)) + if nights["broken"].OK || !nights["mail"].OK { + t.Errorf("recorded %+v", nights) + } +} + +func TestARepositoryThatWillNotOpenIsNeverReplaced(t *testing.T) { + var calls []string + run := func(_ context.Context, _ string, args ...string) (string, error) { + calls = append(calls, strings.Join(args[5:], " ")) + if args[5] == "cat" { + return "", errors.New("Fatal: wrong password or no key found") + } + return "", nil + } + b := &Backups{Where: placed(t, "# pg\npath /\n"), Run: run, Now: time.Now, Say: quiet} + if _, err := b.BackUp(context.Background(), ""); err == nil || !strings.Contains(err.Error(), "cannot be opened, and is left as it is") { + t.Fatalf("got %v", err) + } + for _, c := range calls { + if c == "init" { + t.Fatal("it made a new repository over one it could not open") + } + } +} + +func TestADeclaredDirectoryThatDoesNotExistFailsThatModulesNight(t *testing.T) { + b := &Backups{Where: placed(t, "# pg\npath /nowhere/at/all\n"), Run: func(context.Context, string, ...string) (string, error) { return "", nil }, + Now: time.Now, Say: quiet} + outcome, err := b.BackUp(context.Background(), "") + must(t, err) + if outcome["pg"].OK || !strings.Contains(outcome["pg"].Error, "/nowhere/at/all does not exist") { + t.Fatalf("pg: %+v", outcome["pg"]) + } +} + +func TestANightIsDueAtTheHourAndAMissedOneIsNoticed(t *testing.T) { + morning := time.Date(2026, 10, 6, 1, 30, 0, 0, time.Local) + if got := nextNight(morning, 3); !got.Equal(time.Date(2026, 10, 6, 3, 0, 0, 0, time.Local)) { + t.Errorf("from the morning: %v", got) + } + afternoon := time.Date(2026, 10, 6, 15, 0, 0, 0, time.Local) + if got := nextNight(afternoon, 3); !got.Equal(time.Date(2026, 10, 7, 3, 0, 0, 0, time.Local)) { + t.Errorf("from the afternoon: %v", got) + } + at := func(day int, ok bool) map[string]Night { + return map[string]Night{"pg": {OK: ok, At: time.Date(2026, 10, day, 3, 5, 0, 0, time.Local)}} + } + for _, c := range []struct { + nights map[string]Night + missed bool + }{{map[string]Night{}, true}, {at(6, true), false}, {at(4, true), true}, {at(6, false), true}} { + if got := missedANight(c.nights, afternoon); got != c.missed { + t.Errorf("%+v: missed %v", c.nights, got) + } + } +} + +// The real thing, where restic is installed: a dump, a snapshot, a mistake, and a restore beside. +func TestWithTheRealResticAMistakeIsUndoneBesideTheLiveData(t *testing.T) { + if _, err := exec.LookPath("restic"); err != nil { + t.Skip("restic is not installed") + } + root := t.TempDir() + store, dumps := filepath.Join(root, "store"), filepath.Join(root, "dumps") + must(t, os.Mkdir(store, 0o700)) + must(t, os.Mkdir(dumps, 0o700)) + must(t, os.WriteFile(filepath.Join(store, "mailbox"), []byte("the only copy of a letter\n"), 0o600)) + where := placed(t, "# mail\npath "+store+"\n# pg\nrun echo 'every row' > "+dumps+"/all.dump\npath "+dumps+"\n") + // As whoever runs the test, against its own repository: no sudo. + run := func(ctx context.Context, name string, args ...string) (string, error) { + out, err := exec.CommandContext(ctx, name, args...).Output() + if ee, ok := err.(*exec.ExitError); ok { + return string(out), errors.New(string(ee.Stderr)) + } + return string(out), err + } + b := &Backups{Where: where, Run: run, Now: func() time.Time { return time.Date(2026, 10, 6, 3, 0, 0, 0, time.UTC) }, Say: quiet} + ctx := context.Background() + + night, err := b.BackUp(ctx, "") + must(t, err) + if !night["mail"].OK || !night["pg"].OK { + t.Fatalf("the night: %+v", night) + } + if raw, _ := os.ReadFile(filepath.Join(dumps, "all.dump")); string(raw) != "every row\n" { + t.Fatalf("the dump: %q", raw) + } + + // The mistake. + must(t, os.Remove(filepath.Join(store, "mailbox"))) + + listed, err := b.BackedUp(ctx, "") + must(t, err) + if len(listed) != 2 || listed[0].RestorePoints != 1 || listed[1].RestorePoints != 1 { + t.Fatalf("listed %+v", listed) + } + + restored, err := b.Restore(ctx, "mail", "", "") + must(t, err) + if len(restored.Restored) != 1 || !strings.HasSuffix(restored.Restored[0], "store.restored-20261006-030000") { + t.Fatalf("restored %+v", restored) + } + if raw, _ := os.ReadFile(filepath.Join(restored.Restored[0], "mailbox")); string(raw) != "the only copy of a letter\n" { + t.Fatalf("the restored letter: %q", raw) + } + if _, err := os.Stat(filepath.Join(store, "mailbox")); err == nil { + t.Fatal("the restore wrote into the live directory") + } + + // Never over anything: the same restore again finds its target taken. + if _, err := b.Restore(ctx, "mail", "", ""); err == nil || !strings.Contains(err.Error(), "nothing is restored over anything") { + t.Fatalf("a second restore: %v", err) + } +} diff --git a/modules/restic/cmd/restic-backups/main.go b/modules/restic/cmd/restic-backups/main.go new file mode 100644 index 0000000..9dc24f9 --- /dev/null +++ b/modules/restic/cmd/restic-backups/main.go @@ -0,0 +1,137 @@ +// restic-backups (novox/hq ADR 0214, to-be 43): the machine's backups. One binary, launched by the +// machine's tool runtime and speaking MCP to it over stdio through the Go SDK (ADR 0193, ADR 0198). It +// serves the node-backup seat's three verbs — what is backed up, take one now, restore beside the +// live data — and, beside them, runs the night that happens without anyone asking. +// +// stdout is the MCP channel; everything this module says, it says on stderr. +package main + +import ( + "context" + "fmt" + "os" + "strconv" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// Seat is the role this module holds. +const Seat = "node-backup" + +func say(format string, args ...any) { + fmt.Fprintf(os.Stderr, "[restic] "+format+"\n", args...) +} + +func main() { + where, err := whereFromEnv() + if err != nil { + say("%v", err) + os.Exit(1) + } + b := &Backups{Where: where, Run: execRunner, Now: time.Now, Say: say} + go nights(b) + if err := stdio.Serve("", tools(b)); err != nil { + say("%v", err) + os.Exit(1) + } +} + +// nights runs at the hour, every module; and at start, if a night was missed — the machine was off +// or this module was not running at the hour — one soon rather than a day later. Not at once: a +// machine just started has its stores still coming up. +func nights(b *Backups) { + hour := 3 + if h, err := strconv.Atoi(os.Getenv("MESH_BACKUP_HOUR")); err == nil && h >= 0 && h < 24 { + hour = h + } + if missedANight(b.Nights(), time.Now()) { + time.Sleep(10 * time.Minute) + night(b) + } + for { + time.Sleep(time.Until(nextNight(time.Now(), hour))) + night(b) + } +} + +func night(b *Backups) { + ctx := context.Background() + outcome, err := b.BackUp(ctx, "") + if err != nil { + say("the night did not run: %v", err) + return + } + var failed []string + for module, n := range outcome { + if !n.OK { + failed = append(failed, module) + } + } + if len(failed) > 0 { + say("the night left %s without a backup", strings.Join(failed, ", ")) + } + // Sundays, the repository's own integrity with a sample of the data read back. + if time.Now().Weekday() == time.Sunday { + if err := b.Check(ctx); err != nil { + say("the repository does NOT check out: %v", err) + } else { + say("the repository checks out") + } + } +} + +// ---- the seat's verbs -------------------------------------------------------------------------- + +func str(description string) map[string]any { + return map[string]any{"type": "string", "description": description} +} + +func arg(a map[string]any, k string) string { + v, _ := a[k].(string) + return strings.TrimSpace(v) +} + +// verb is one of the seat's verbs: listed as `.`, so the runtime serves it on the seat's +// subject. +func verb(name, description string, input map[string]any, run func(a map[string]any) (any, error)) stdio.Tool { + return stdio.Tool{Name: Seat + "." + name, Description: description, Input: input, Run: run} +} + +func tools(b *Backups) []stdio.Tool { + return []stdio.Tool{ + verb("backed-up", "What this machine backs up: each module, what it declared, its last good night, how many restore points are kept.", + map[string]any{"module": str("one module (optional)")}, + func(a map[string]any) (any, error) { return b.BackedUp(context.Background(), arg(a, "module")) }), + verb("now", "Take a backup now, of one module or of every module on this machine — before a migration, a retirement or anything else that could go wrong. Answers when it has started; `backed-up` says how it went.", + map[string]any{"module": str("one module (optional)")}, + func(a map[string]any) (any, error) { + only := arg(a, "module") + // A night of a large store outlasts any call; it is started, and its outcome recorded. + go func() { + if _, err := b.BackUp(context.Background(), only); err != nil { + say("a backup asked for now failed: %v", err) + } + }() + started := only + if started == "" { + started = "every module on this machine" + } + return map[string]any{"started": started, "follow": Seat + ".backed-up"}, nil + }), + verb("restore", "Restore one module's data from a restore point BESIDE the live data, never over it: each directory as .restored-. Swapping it in is a person's act.", + map[string]any{ + "module": str("the module"), + "snapshot": str("the restore point (the newest when omitted)"), + "path": str("one of the module's directories (all of them when omitted)"), + }, + func(a map[string]any) (any, error) { + module := arg(a, "module") + if module == "" { + return nil, fmt.Errorf("module is required") + } + return b.Restore(context.Background(), module, arg(a, "snapshot"), arg(a, "path")) + }), + } +} diff --git a/modules/restic/go.mod b/modules/restic/go.mod new file mode 100644 index 0000000..f010f2f --- /dev/null +++ b/modules/restic/go.mod @@ -0,0 +1,5 @@ +module restic + +go 1.25.0 + +require git.novox.be/novox/mesh-sdk/go v0.1.7 diff --git a/modules/restic/go.sum b/modules/restic/go.sum new file mode 100644 index 0000000..b474419 --- /dev/null +++ b/modules/restic/go.sum @@ -0,0 +1,2 @@ +git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w= +git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= diff --git a/modules/restic/module.json b/modules/restic/module.json new file mode 100644 index 0000000..f304b89 --- /dev/null +++ b/modules/restic/module.json @@ -0,0 +1,64 @@ +{ + "module": "restic", + "version": "1", + "claims": [ + { + "name": "node-backup", + "scope": "node", + "serves": [ + "backed-up", + "now", + "restore" + ] + } + ], + "own-secrets": { + "repository": "${dir:state}/repository.secret" + }, + "resources": [ + { + "id": "state", + "type": "directory", + "mode": "0700", + "place": "." + }, + { + "id": "repository", + "type": "directory", + "mode": "0700" + }, + { + "id": "declared", + "type": "file", + "path": "${dir:state}/backups.conf", + "mode": "0600", + "content": "# What the modules on this machine back up, composed by the mesh (novox/hq to-be 43). Do not edit.\n${contribution:node-backup:backup}" + }, + { + "id": "tool", + "type": "package", + "package": "restic" + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/restic-backups", + "binary": "restic-backups", + "loads": [ + "restic-backups" + ], + "env": { + "MESH_BACKUP_DECLARED": "${dir:state}/backups.conf", + "MESH_BACKUP_REPOSITORY": "${dir:repository}", + "MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret", + "MESH_BACKUP_STATE": "${dir:state}" + } + } + ] + } +}