jschoubben
46e3a5a6b7
The first restore of an arr app refused its settings file with "not a directory": restic restores a snapshot's subfolder, not a file. A file is restored with its full path into a scratch directory beside the target, moved into place, and the scratch removed.
287 lines
11 KiB
Go
287 lines
11 KiB
Go
package main
|
|
|
|
// The machine's backups over a fake restic and fake stores (novox/hq ADR 0214, to-be 43), and — where
|
|
// restic is installed — over the real one, on throwaway directories.
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
const composed = "# What the modules on this machine back up, composed by the mesh. Do not edit.\n" +
|
|
"# postgres\nrun docker exec -u postgres postgres sh -c 'pg-dump-all'\npath /var/lib/mesh-store/dumps\n" +
|
|
"# mailu\npath /var/lib/mailu/data-mail\npath /var/lib/mailu/data-dkim\n"
|
|
|
|
func placed(t *testing.T, declared string) Where {
|
|
t.Helper()
|
|
dir := t.TempDir()
|
|
must(t, os.WriteFile(filepath.Join(dir, "backups.conf"), []byte(declared), 0o600))
|
|
must(t, os.WriteFile(filepath.Join(dir, "pw"), []byte("secret\n"), 0o600))
|
|
return Where{Declared: filepath.Join(dir, "backups.conf"), Repository: filepath.Join(dir, "repo"),
|
|
PasswordFile: filepath.Join(dir, "pw"), State: dir}
|
|
}
|
|
|
|
func must(t *testing.T, err error) {
|
|
t.Helper()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func quiet(string, ...any) {}
|
|
|
|
func TestTheComposedFileIsReadIntoEachModulesRunsAndPaths(t *testing.T) {
|
|
got, err := parseDeclared(composed)
|
|
must(t, err)
|
|
want := []Declared{
|
|
{Module: "postgres", Runs: []string{"docker exec -u postgres postgres sh -c 'pg-dump-all'"}, Paths: []string{"/var/lib/mesh-store/dumps"}},
|
|
{Module: "mailu", Paths: []string{"/var/lib/mailu/data-mail", "/var/lib/mailu/data-dkim"}},
|
|
}
|
|
if !reflect.DeepEqual(got, want) {
|
|
t.Fatalf("read %#v, want %#v", got, want)
|
|
}
|
|
}
|
|
|
|
func TestALineThisHolderDoesNotReadIsRefusedNamingTheModule(t *testing.T) {
|
|
for _, bad := range []string{"# pg\ncopy /x\n", "# pg\npath relative/dir\n"} {
|
|
if _, err := parseDeclared(bad); err == nil || !strings.Contains(err.Error(), "pg contributes a backup line") {
|
|
t.Errorf("%q: %v", bad, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestResticAndTheDumpsRunThroughSudoWhereTheAccountIsNotRoot(t *testing.T) {
|
|
if p, a := escalated(1000, "restic", []string{"snapshots"}); p != "sudo" || !reflect.DeepEqual(a, []string{"-n", "restic", "snapshots"}) {
|
|
t.Errorf("as an account: %s %v", p, a)
|
|
}
|
|
if p, a := escalated(0, "restic", []string{"snapshots"}); p != "restic" || !reflect.DeepEqual(a, []string{"snapshots"}) {
|
|
t.Errorf("as root: %s %v", p, a)
|
|
}
|
|
}
|
|
|
|
func TestANightDumpsBeforeEachSnapshotAndOneFailingModuleFailsOnlyItself(t *testing.T) {
|
|
where := placed(t, "# pg\nrun dump-it\npath /\n# broken\nrun fail-it\npath /\n# mail\npath /\n")
|
|
var calls []string
|
|
run := func(_ context.Context, name string, args ...string) (string, error) {
|
|
line := name + " " + strings.Join(args, " ")
|
|
if name == "restic" {
|
|
line = "restic " + strings.Join(args[5:], " ")
|
|
}
|
|
if name == "test" {
|
|
return "", nil
|
|
}
|
|
calls = append(calls, line)
|
|
switch {
|
|
case line == "sh -c fail-it":
|
|
return "", errors.New("the dump failed")
|
|
case strings.HasPrefix(line, "restic backup"):
|
|
return "{\"message_type\":\"status\"}\n{\"message_type\":\"summary\",\"snapshot_id\":\"abcdef0123456789\"}\n", nil
|
|
}
|
|
return "", nil
|
|
}
|
|
b := &Backups{Where: where, Run: run, Now: func() time.Time { return time.Date(2026, 10, 6, 3, 0, 0, 0, time.UTC) }, Say: quiet}
|
|
outcome, err := b.BackUp(context.Background(), "")
|
|
must(t, err)
|
|
if !outcome["pg"].OK || outcome["pg"].Snapshot != "abcdef01" {
|
|
t.Errorf("pg: %+v", outcome["pg"])
|
|
}
|
|
if outcome["broken"].OK || !strings.Contains(outcome["broken"].Error, "the dump failed") {
|
|
t.Errorf("broken: %+v", outcome["broken"])
|
|
}
|
|
if !outcome["mail"].OK {
|
|
t.Errorf("mail failed with broken: %+v", outcome["mail"])
|
|
}
|
|
want := []string{
|
|
"restic cat config",
|
|
"sh -c dump-it",
|
|
"restic backup --json --tag module=pg /",
|
|
"sh -c fail-it",
|
|
"restic backup --json --tag module=mail /",
|
|
"restic forget --prune --group-by host,tags --keep-daily 14 --keep-weekly 8 --keep-monthly 6",
|
|
}
|
|
if !reflect.DeepEqual(calls, want) {
|
|
t.Errorf("ran\n%s\nwant\n%s", strings.Join(calls, "\n"), strings.Join(want, "\n"))
|
|
}
|
|
// Recorded, so `backed-up` and the missed-night check read it.
|
|
var nights map[string]Night
|
|
raw, err := os.ReadFile(filepath.Join(where.State, "nights.json"))
|
|
must(t, err)
|
|
must(t, json.Unmarshal(raw, &nights))
|
|
if nights["broken"].OK || !nights["mail"].OK {
|
|
t.Errorf("recorded %+v", nights)
|
|
}
|
|
}
|
|
|
|
func TestARepositoryThatWillNotOpenIsNeverReplaced(t *testing.T) {
|
|
var calls []string
|
|
run := func(_ context.Context, _ string, args ...string) (string, error) {
|
|
calls = append(calls, strings.Join(args[5:], " "))
|
|
if args[5] == "cat" {
|
|
return "", errors.New("Fatal: wrong password or no key found")
|
|
}
|
|
return "", nil
|
|
}
|
|
b := &Backups{Where: placed(t, "# pg\npath /\n"), Run: run, Now: time.Now, Say: quiet}
|
|
if _, err := b.BackUp(context.Background(), ""); err == nil || !strings.Contains(err.Error(), "cannot be opened, and is left as it is") {
|
|
t.Fatalf("got %v", err)
|
|
}
|
|
for _, c := range calls {
|
|
if c == "init" {
|
|
t.Fatal("it made a new repository over one it could not open")
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestADeclaredDirectoryThatDoesNotExistFailsThatModulesNight(t *testing.T) {
|
|
run := func(_ context.Context, name string, args ...string) (string, error) {
|
|
if name == "test" && args[1] == "/nowhere/at/all" {
|
|
return "", errors.New("exit status 1")
|
|
}
|
|
return "", nil
|
|
}
|
|
b := &Backups{Where: placed(t, "# pg\npath /nowhere/at/all\n"), Run: run, Now: time.Now, Say: quiet}
|
|
outcome, err := b.BackUp(context.Background(), "")
|
|
must(t, err)
|
|
if outcome["pg"].OK || !strings.Contains(outcome["pg"].Error, "/nowhere/at/all does not exist") {
|
|
t.Fatalf("pg: %+v", outcome["pg"])
|
|
}
|
|
}
|
|
|
|
func TestANightIsDueAtTheHourAndAMissedOneIsNoticed(t *testing.T) {
|
|
morning := time.Date(2026, 10, 6, 1, 30, 0, 0, time.Local)
|
|
if got := nextNight(morning, 3); !got.Equal(time.Date(2026, 10, 6, 3, 0, 0, 0, time.Local)) {
|
|
t.Errorf("from the morning: %v", got)
|
|
}
|
|
afternoon := time.Date(2026, 10, 6, 15, 0, 0, 0, time.Local)
|
|
if got := nextNight(afternoon, 3); !got.Equal(time.Date(2026, 10, 7, 3, 0, 0, 0, time.Local)) {
|
|
t.Errorf("from the afternoon: %v", got)
|
|
}
|
|
at := func(day int, ok bool) map[string]Night {
|
|
return map[string]Night{"pg": {OK: ok, At: time.Date(2026, 10, day, 3, 5, 0, 0, time.Local)}}
|
|
}
|
|
for _, c := range []struct {
|
|
nights map[string]Night
|
|
missed bool
|
|
}{{map[string]Night{}, true}, {at(6, true), false}, {at(4, true), true}, {at(6, false), true}} {
|
|
if got := missedANight(c.nights, afternoon); got != c.missed {
|
|
t.Errorf("%+v: missed %v", c.nights, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
// The real thing, where restic is installed: a dump, a snapshot, a mistake, and a restore beside.
|
|
func TestWithTheRealResticAMistakeIsUndoneBesideTheLiveData(t *testing.T) {
|
|
if _, err := exec.LookPath("restic"); err != nil {
|
|
t.Skip("restic is not installed")
|
|
}
|
|
root := t.TempDir()
|
|
store, dumps := filepath.Join(root, "store"), filepath.Join(root, "dumps")
|
|
must(t, os.Mkdir(store, 0o700))
|
|
must(t, os.Mkdir(dumps, 0o700))
|
|
must(t, os.WriteFile(filepath.Join(store, "mailbox"), []byte("the only copy of a letter\n"), 0o600))
|
|
settings := filepath.Join(root, "settings.xml")
|
|
must(t, os.WriteFile(settings, []byte("<Config>kept</Config>\n"), 0o600))
|
|
where := placed(t, "# mail\npath "+store+"\npath "+settings+"\n# pg\nrun echo 'every row' > "+dumps+"/all.dump\npath "+dumps+"\n")
|
|
// As whoever runs the test, against its own repository: no sudo.
|
|
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
|
out, err := exec.CommandContext(ctx, name, args...).Output()
|
|
if ee, ok := err.(*exec.ExitError); ok {
|
|
return string(out), errors.New(string(ee.Stderr))
|
|
}
|
|
return string(out), err
|
|
}
|
|
b := &Backups{Where: where, Run: run, Now: func() time.Time { return time.Date(2026, 10, 6, 3, 0, 0, 0, time.UTC) }, Say: quiet}
|
|
ctx := context.Background()
|
|
|
|
night, err := b.BackUp(ctx, "")
|
|
must(t, err)
|
|
if !night["mail"].OK || !night["pg"].OK {
|
|
t.Fatalf("the night: %+v", night)
|
|
}
|
|
if raw, _ := os.ReadFile(filepath.Join(dumps, "all.dump")); string(raw) != "every row\n" {
|
|
t.Fatalf("the dump: %q", raw)
|
|
}
|
|
|
|
// The mistake.
|
|
must(t, os.Remove(filepath.Join(store, "mailbox")))
|
|
|
|
listed, err := b.BackedUp(ctx, "")
|
|
must(t, err)
|
|
if len(listed) != 2 || listed[0].RestorePoints != 1 || listed[1].RestorePoints != 1 {
|
|
t.Fatalf("listed %+v", listed)
|
|
}
|
|
|
|
// The mistake to a single file, too.
|
|
must(t, os.WriteFile(settings, []byte("<Config>overwritten</Config>\n"), 0o600))
|
|
|
|
restored, err := b.Restore(ctx, "mail", "", "")
|
|
must(t, err)
|
|
if len(restored.Restored) != 2 {
|
|
t.Fatalf("restored %+v", restored)
|
|
}
|
|
var dir, file string
|
|
for _, r := range restored.Restored {
|
|
switch {
|
|
case strings.HasSuffix(r, "store.restored-20261006-030000"):
|
|
dir = r
|
|
case strings.HasSuffix(r, "settings.xml.restored-20261006-030000"):
|
|
file = r
|
|
}
|
|
}
|
|
if raw, _ := os.ReadFile(filepath.Join(dir, "mailbox")); string(raw) != "the only copy of a letter\n" {
|
|
t.Fatalf("the restored letter: %q", raw)
|
|
}
|
|
// A single file comes back as a file beside the live one, and nothing of the scratch remains.
|
|
if raw, _ := os.ReadFile(file); string(raw) != "<Config>kept</Config>\n" {
|
|
t.Fatalf("the restored settings: %q", raw)
|
|
}
|
|
if raw, _ := os.ReadFile(settings); string(raw) != "<Config>overwritten</Config>\n" {
|
|
t.Fatalf("the restore wrote over the live settings: %q", raw)
|
|
}
|
|
if _, err := os.Stat(file + ".partial"); err == nil {
|
|
t.Fatal("the scratch directory was left behind")
|
|
}
|
|
if _, err := os.Stat(filepath.Join(store, "mailbox")); err == nil {
|
|
t.Fatal("the restore wrote into the live directory")
|
|
}
|
|
|
|
// Never over anything: the same restore again finds its target taken.
|
|
if _, err := b.Restore(ctx, "mail", "", ""); err == nil || !strings.Contains(err.Error(), "nothing is restored over anything") {
|
|
t.Fatalf("a second restore: %v", err)
|
|
}
|
|
}
|
|
|
|
// A store's directory is often its own user's alone; whether it is there is asked as root, never by
|
|
// the runtime's account looking for itself — which saw nothing in postgres's 0700 data directory and
|
|
// called the dumps missing on the first run (2026-10-05).
|
|
func TestWhetherADirectoryIsThereIsAskedAsRoot(t *testing.T) {
|
|
var asked []string
|
|
run := func(_ context.Context, name string, args ...string) (string, error) {
|
|
if name == "test" {
|
|
asked = append(asked, strings.Join(args, " "))
|
|
}
|
|
if name == "restic" && args[5] == "backup" {
|
|
return "{\"message_type\":\"summary\",\"snapshot_id\":\"0123456789abcdef\"}\n", nil
|
|
}
|
|
return "", nil
|
|
}
|
|
// A path the account cannot see, which root can.
|
|
b := &Backups{Where: placed(t, "# pg\npath /root/only/dumps\n"), Run: run, Now: time.Now, Say: quiet}
|
|
outcome, err := b.BackUp(context.Background(), "")
|
|
must(t, err)
|
|
if !outcome["pg"].OK {
|
|
t.Fatalf("a directory only root sees was called missing: %+v", outcome["pg"])
|
|
}
|
|
if !reflect.DeepEqual(asked, []string{"-e /root/only/dumps"}) {
|
|
t.Errorf("asked %v", asked)
|
|
}
|
|
}
|