mesh-catalog/modules/restic/cmd/restic-backups/backups_test.go

package main

// The machine's backups over a fake restic and fake stores (novox/hq ADR 0214, to-be 43), and — where
// restic is installed — over the real one, on throwaway directories.

import (
	"context"
	"encoding/json"
	"errors"
	"os"
	"os/exec"
	"path/filepath"
	"reflect"
	"strings"
	"testing"
	"time"
)

const composed = "# What the modules on this machine back up, composed by the mesh. Do not edit.\n" +
	"# postgres\nrun docker exec -u postgres postgres sh -c 'pg-dump-all'\npath /var/lib/mesh-store/dumps\n" +
	"# mailu\npath /var/lib/mailu/data-mail\npath /var/lib/mailu/data-dkim\n"

func placed(t *testing.T, declared string) Where {
	t.Helper()
	dir := t.TempDir()
	must(t, os.WriteFile(filepath.Join(dir, "backups.conf"), []byte(declared), 0o600))
	must(t, os.WriteFile(filepath.Join(dir, "pw"), []byte("secret\n"), 0o600))
	// The retention file every machine has, as the mesh writes it with the defaults (novox/hq issue 454).
	must(t, os.WriteFile(filepath.Join(dir, RetentionFile), []byte("daily=14\nweekly=8\nmonthly=6\n"), 0o600))
	return Where{Declared: filepath.Join(dir, "backups.conf"), Repository: filepath.Join(dir, "repo"),
		PasswordFile: filepath.Join(dir, "pw"), State: dir}
}

func must(t *testing.T, err error) {
	t.Helper()
	if err != nil {
		t.Fatal(err)
	}
}

func quiet(string, ...any) {}

func TestTheComposedFileIsReadIntoEachModulesRunsAndPaths(t *testing.T) {
	got, err := parseDeclared(composed)
	must(t, err)
	want := []Declared{
		{Module: "postgres", Runs: []string{"docker exec -u postgres postgres sh -c 'pg-dump-all'"}, Paths: []string{"/var/lib/mesh-store/dumps"}},
		{Module: "mailu", Paths: []string{"/var/lib/mailu/data-mail", "/var/lib/mailu/data-dkim"}},
	}
	if !reflect.DeepEqual(got, want) {
		t.Fatalf("read %#v, want %#v", got, want)
	}
}

func TestALineThisHolderDoesNotReadIsRefusedNamingTheModule(t *testing.T) {
	for _, bad := range []string{"# pg\ncopy /x\n", "# pg\npath relative/dir\n"} {
		if _, err := parseDeclared(bad); err == nil || !strings.Contains(err.Error(), "pg contributes a backup line") {
			t.Errorf("%q: %v", bad, err)
		}
	}
}

func TestResticAndTheDumpsRunThroughSudoWhereTheAccountIsNotRoot(t *testing.T) {
	if p, a := escalated(1000, "restic", []string{"snapshots"}); p != "sudo" || !reflect.DeepEqual(a, []string{"-n", "restic", "snapshots"}) {
		t.Errorf("as an account: %s %v", p, a)
	}
	if p, a := escalated(0, "restic", []string{"snapshots"}); p != "restic" || !reflect.DeepEqual(a, []string{"snapshots"}) {
		t.Errorf("as root: %s %v", p, a)
	}
}

func TestANightDumpsBeforeEachSnapshotAndOneFailingModuleFailsOnlyItself(t *testing.T) {
	where := placed(t, "# pg\nrun dump-it\npath /\n# broken\nrun fail-it\npath /\n# mail\npath /\n")
	var calls []string
	run := func(_ context.Context, name string, args ...string) (string, error) {
		line := name + " " + strings.Join(args, " ")
		if name == "restic" {
			line = "restic " + strings.Join(args[5:], " ")
		}
		if name == "test" {
			return "", nil
		}
		calls = append(calls, line)
		switch {
		case line == "sh -c fail-it":
			return "", errors.New("the dump failed")
		case strings.HasPrefix(line, "restic backup"):
			return "{\"message_type\":\"status\"}\n{\"message_type\":\"summary\",\"snapshot_id\":\"abcdef0123456789\"}\n", nil
		}
		return "", nil
	}
	b := &Backups{Where: where, Run: run, Now: func() time.Time { return time.Date(2026, 10, 6, 3, 0, 0, 0, time.UTC) }, Say: quiet}
	outcome, err := b.BackUp(context.Background(), "")
	must(t, err)
	if !outcome["pg"].OK || outcome["pg"].Snapshot != "abcdef01" {
		t.Errorf("pg: %+v", outcome["pg"])
	}
	if outcome["broken"].OK || !strings.Contains(outcome["broken"].Error, "the dump failed") {
		t.Errorf("broken: %+v", outcome["broken"])
	}
	if !outcome["mail"].OK {
		t.Errorf("mail failed with broken: %+v", outcome["mail"])
	}
	want := []string{
		"restic cat config",
		"sh -c dump-it",
		"restic backup --json --tag module=pg /",
		"sh -c fail-it",
		"restic backup --json --tag module=mail /",
		"restic forget --prune --group-by host,tags --keep-daily 14 --keep-weekly 8 --keep-monthly 6",
	}
	if !reflect.DeepEqual(calls, want) {
		t.Errorf("ran\n%s\nwant\n%s", strings.Join(calls, "\n"), strings.Join(want, "\n"))
	}
	// Recorded, so `backed-up` and the missed-night check read it.
	var nights map[string]Night
	raw, err := os.ReadFile(filepath.Join(where.State, "nights.json"))
	must(t, err)
	must(t, json.Unmarshal(raw, &nights))
	if nights["broken"].OK || !nights["mail"].OK {
		t.Errorf("recorded %+v", nights)
	}
}

func TestARepositoryThatWillNotOpenIsNeverReplaced(t *testing.T) {
	var calls []string
	run := func(_ context.Context, _ string, args ...string) (string, error) {
		calls = append(calls, strings.Join(args[5:], " "))
		if args[5] == "cat" {
			return "", errors.New("Fatal: wrong password or no key found")
		}
		return "", nil
	}
	b := &Backups{Where: placed(t, "# pg\npath /\n"), Run: run, Now: time.Now, Say: quiet}
	if _, err := b.BackUp(context.Background(), ""); err == nil || !strings.Contains(err.Error(), "cannot be opened, and is left as it is") {
		t.Fatalf("got %v", err)
	}
	for _, c := range calls {
		if c == "init" {
			t.Fatal("it made a new repository over one it could not open")
		}
	}
}

func TestADeclaredDirectoryThatDoesNotExistFailsThatModulesNight(t *testing.T) {
	run := func(_ context.Context, name string, args ...string) (string, error) {
		if name == "test" && args[1] == "/nowhere/at/all" {
			return "", errors.New("exit status 1")
		}
		return "", nil
	}
	b := &Backups{Where: placed(t, "# pg\npath /nowhere/at/all\n"), Run: run, Now: time.Now, Say: quiet}
	outcome, err := b.BackUp(context.Background(), "")
	must(t, err)
	if outcome["pg"].OK || !strings.Contains(outcome["pg"].Error, "/nowhere/at/all does not exist") {
		t.Fatalf("pg: %+v", outcome["pg"])
	}
}

func TestANightIsDueAtTheHourAndAMissedOneIsNoticed(t *testing.T) {
	morning := time.Date(2026, 10, 6, 1, 30, 0, 0, time.Local)
	if got := nextNight(morning, 3); !got.Equal(time.Date(2026, 10, 6, 3, 0, 0, 0, time.Local)) {
		t.Errorf("from the morning: %v", got)
	}
	afternoon := time.Date(2026, 10, 6, 15, 0, 0, 0, time.Local)
	if got := nextNight(afternoon, 3); !got.Equal(time.Date(2026, 10, 7, 3, 0, 0, 0, time.Local)) {
		t.Errorf("from the afternoon: %v", got)
	}
	at := func(day int, ok bool) map[string]Night {
		return map[string]Night{"pg": {OK: ok, At: time.Date(2026, 10, day, 3, 5, 0, 0, time.Local)}}
	}
	for _, c := range []struct {
		nights map[string]Night
		missed bool
	}{{map[string]Night{}, true}, {at(6, true), false}, {at(4, true), true}, {at(6, false), true}} {
		if got := missedANight(c.nights, afternoon); got != c.missed {
			t.Errorf("%+v: missed %v", c.nights, got)
		}
	}
}

// The real thing, where restic is installed: a dump, a snapshot, a mistake, and a restore beside.
func TestWithTheRealResticAMistakeIsUndoneBesideTheLiveData(t *testing.T) {
	if _, err := exec.LookPath("restic"); err != nil {
		t.Skip("restic is not installed")
	}
	root := t.TempDir()
	store, dumps := filepath.Join(root, "store"), filepath.Join(root, "dumps")
	must(t, os.Mkdir(store, 0o700))
	must(t, os.Mkdir(dumps, 0o700))
	must(t, os.WriteFile(filepath.Join(store, "mailbox"), []byte("the only copy of a letter\n"), 0o600))
	settings := filepath.Join(root, "settings.xml")
	must(t, os.WriteFile(settings, []byte("<Config>kept</Config>\n"), 0o600))
	where := placed(t, "# mail\npath "+store+"\npath "+settings+"\n# pg\nrun echo 'every row' > "+dumps+"/all.dump\npath "+dumps+"\n")
	// As whoever runs the test, against its own repository: no sudo. The test's directory lies in a temporary
	// directory everyone may write, which a real restore is right to refuse beside; the directories above the
	// test's own are said to be root's here, so the restore beside is what is tested.
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		if name == "stat" && len(args) > 0 && strings.HasPrefix(root, strings.TrimSuffix(args[len(args)-1], "/")+"/") {
			return "0 755 directory\n", nil
		}
		out, err := exec.CommandContext(ctx, name, args...).Output()
		if ee, ok := err.(*exec.ExitError); ok {
			return string(out), errors.New(string(ee.Stderr))
		}
		return string(out), err
	}
	b := &Backups{Where: where, Run: run, Now: func() time.Time { return time.Date(2026, 10, 6, 3, 0, 0, 0, time.UTC) }, Say: quiet}
	ctx := context.Background()

	night, err := b.BackUp(ctx, "")
	must(t, err)
	if !night["mail"].OK || !night["pg"].OK {
		t.Fatalf("the night: %+v", night)
	}
	if raw, _ := os.ReadFile(filepath.Join(dumps, "all.dump")); string(raw) != "every row\n" {
		t.Fatalf("the dump: %q", raw)
	}

	// The mistake.
	must(t, os.Remove(filepath.Join(store, "mailbox")))

	listed, err := b.BackedUp(ctx, "")
	must(t, err)
	if len(listed) != 2 || listed[0].RestorePoints != 1 || listed[1].RestorePoints != 1 {
		t.Fatalf("listed %+v", listed)
	}

	// The mistake to a single file, too.
	must(t, os.WriteFile(settings, []byte("<Config>overwritten</Config>\n"), 0o600))

	restored, err := b.Restore(ctx, "mail", "", "")
	must(t, err)
	if len(restored.Restored) != 2 {
		t.Fatalf("restored %+v", restored)
	}
	var dir, file string
	for _, r := range restored.Restored {
		switch {
		case strings.HasSuffix(r, "store.restored-20261006-030000"):
			dir = r
		case strings.HasSuffix(r, "settings.xml.restored-20261006-030000"):
			file = r
		}
	}
	if raw, _ := os.ReadFile(filepath.Join(dir, "mailbox")); string(raw) != "the only copy of a letter\n" {
		t.Fatalf("the restored letter: %q", raw)
	}
	// A single file comes back as a file beside the live one, and nothing of the scratch remains.
	if raw, _ := os.ReadFile(file); string(raw) != "<Config>kept</Config>\n" {
		t.Fatalf("the restored settings: %q", raw)
	}
	if raw, _ := os.ReadFile(settings); string(raw) != "<Config>overwritten</Config>\n" {
		t.Fatalf("the restore wrote over the live settings: %q", raw)
	}
	if _, err := os.Stat(file + ".partial"); err == nil {
		t.Fatal("the scratch directory was left behind")
	}
	if _, err := os.Stat(filepath.Join(store, "mailbox")); err == nil {
		t.Fatal("the restore wrote into the live directory")
	}

	// Never over anything: the same restore again finds its target taken.
	if _, err := b.Restore(ctx, "mail", "", ""); err == nil || !strings.Contains(err.Error(), "nothing is restored over anything") {
		t.Fatalf("a second restore: %v", err)
	}
}

// A store's directory is often its own user's alone; whether it is there is asked as root, never by
// the runtime's account looking for itself — which saw nothing in postgres's 0700 data directory and
// called the dumps missing on the first run (2026-10-05).
func TestWhetherADirectoryIsThereIsAskedAsRoot(t *testing.T) {
	var asked []string
	run := func(_ context.Context, name string, args ...string) (string, error) {
		if name == "test" {
			asked = append(asked, strings.Join(args, " "))
		}
		if name == "restic" && args[5] == "backup" {
			return "{\"message_type\":\"summary\",\"snapshot_id\":\"0123456789abcdef\"}\n", nil
		}
		return "", nil
	}
	// A path the account cannot see, which root can.
	b := &Backups{Where: placed(t, "# pg\npath /root/only/dumps\n"), Run: run, Now: time.Now, Say: quiet}
	outcome, err := b.BackUp(context.Background(), "")
	must(t, err)
	if !outcome["pg"].OK {
		t.Fatalf("a directory only root sees was called missing: %+v", outcome["pg"])
	}
	if !reflect.DeepEqual(asked, []string{"-e /root/only/dumps"}) {
		t.Errorf("asked %v", asked)
	}
}

// Beside a directory another account owns, root does not restore: a link swapped in there would point root's
// write anywhere (novox/hq ADR 0266). The restore goes to the module's state instead.
func TestARestoreBesideAnotherAccountsDirectoryGoesToTheState(t *testing.T) {
	state := t.TempDir()
	var ran []string
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		ran = append(ran, name+" "+strings.Join(args, " "))
		if name == "stat" {
			if args[len(args)-1] == "/home/agent" {
				return "1001 700 directory\n", nil
			}
			if args[len(args)-1] == "/srv/shared" {
				return "0 1777 directory\n", nil
			}
			return "0 755 directory\n", nil
		}
		return "", nil
	}
	b := &Backups{Where: Where{State: state}, Run: run, Now: time.Now, Say: func(string, ...any) {}}
	if why := b.besideIsUnsafe(context.Background(), "/home/agent/.claude.restored-x"); !strings.Contains(why, "another account") {
		t.Fatalf("an agent's home is unsafe beside: %q", why)
	}
	if why := b.besideIsUnsafe(context.Background(), "/srv/data.restored-x"); why != "" {
		t.Fatalf("root's directories are safe beside: %q", why)
	}
	if why := b.besideIsUnsafe(context.Background(), "/srv/shared/data.restored-x"); !strings.Contains(why, "other accounts") {
		t.Fatalf("a directory everyone may write is unsafe beside, sticky or not: %q", why)
	}
	run2 := func(ctx context.Context, name string, args ...string) (string, error) {
		if name == "stat" && args[len(args)-1] == "/home" {
			return "0 777 symbolic link\n", nil
		}
		return "0 755 directory\n", nil
	}
	b.Run = run2
	if why := b.besideIsUnsafe(context.Background(), "/home/agent/x"); !strings.Contains(why, "symbolic link") {
		t.Fatalf("a link on the way is unsafe: %q", why)
	}
	_ = ran
}