mesh-catalog/modules/restic/cmd/restic-backups/hand_test.go

package main

// A backup taken by hand — `node-backup.now`, asked by a person or by the mesh before a module moves (novox/hq
// ADR 0233) — is kept by the thinning as its own restore point: until a later good night of the same module
// exists, and for the machine's keep-hand-taken-days in any case (novox/hq issue 453: one taken at 02:35 was
// forgotten by the night at 03:01, as keep-daily keeps only the newest restore point of each day).

import (
	"context"
	"encoding/json"
	"errors"
	"os"
	"os/exec"
	"path/filepath"
	"slices"
	"strings"
	"testing"
	"time"
)

// handRestic is a fake restic that remembers each command and lists the snapshots it is given. A backup makes
// the snapshot made (abcdef01 when empty), and fails as restic's exit 3 when partial.
type handRestic struct {
	calls   []string
	snaps   []Snapshot
	made    string
	partial bool
}

func (h *handRestic) run(_ context.Context, name string, args ...string) (string, error) {
	if name != "restic" {
		return "", nil
	}
	line := "restic " + strings.Join(args[5:], " ")
	h.calls = append(h.calls, line)
	switch args[5] {
	case "backup":
		if h.partial {
			return "", errors.New("Warning: at least one source file could not be read")
		}
		made := h.made
		if made == "" {
			made = "abcdef0123456789"
		}
		return "{\"message_type\":\"summary\",\"snapshot_id\":\"" + made + "\"}\n", nil
	case "snapshots":
		raw, _ := json.Marshal(h.snaps)
		return string(raw), nil
	}
	return "", nil
}

func (h *handRestic) with(prefix string) []string {
	var out []string
	for _, c := range h.calls {
		if strings.HasPrefix(c, prefix) {
			out = append(out, c)
		}
	}
	return out
}

func TestAHandTakenBackupIsTaggedAndTheThinningKeepsThatTag(t *testing.T) {
	h := &handRestic{}
	b := &Backups{Where: placed(t, "# pg\npath /\n"), Run: h.run,
		Now: func() time.Time { return time.Date(2026, 10, 11, 2, 35, 0, 0, time.UTC) }, Say: quiet}
	_, err := b.BackUpNow(context.Background(), "pg")
	must(t, err)
	if got := h.with("restic backup"); len(got) != 1 || got[0] != "restic backup --json --tag module=pg --tag hand /" {
		t.Errorf("backed up by hand with %q, want it tagged hand", got)
	}
	_, err = b.BackUp(context.Background(), "")
	must(t, err)
	if got := h.with("restic backup"); len(got) != 2 || got[1] != "restic backup --json --tag module=pg /" {
		t.Errorf("the night backed up with %q, want it tagged with its module alone", got)
	}
	want := "restic forget --prune --group-by host,tags --keep-daily 14 --keep-weekly 8 --keep-monthly 6 --keep-tag hand --retry-lock 10m0s"
	for _, f := range h.with("restic forget") {
		if f != want {
			t.Errorf("thinned with %q, want %q", f, want)
		}
		// No rule that would keep every night as well: the hand tag is the only thing kept beyond the counts.
		if strings.Contains(f, "--keep-within") || strings.Contains(f, "--keep-last") {
			t.Errorf("thinned with %q, a rule that keeps nights too", f)
		}
	}
	if len(h.with("restic forget")) != 2 {
		t.Errorf("thinned %d times, want after each of the two backups", len(h.with("restic forget")))
	}
}

func TestAHandTakenRestorePointIsLetGoOnlyPastItsBoundAndAfterALaterNight(t *testing.T) {
	now := time.Date(2026, 10, 20, 3, 0, 0, 0, time.UTC)
	at := func(days float64) time.Time { return now.Add(-time.Duration(days * 24 * float64(time.Hour))) }
	snap := func(id string, when time.Time, tags ...string) Snapshot {
		return Snapshot{ID: id + "0000000000", ShortID: id, Time: when, Tags: tags}
	}
	snaps := []Snapshot{
		snap("oldhand1", at(10), "module=pg", HandTag),   // past the bound, a night after it: let go
		snap("night001", at(9), "module=pg"),             // the night after it
		snap("freshhnd", at(2), "module=pg", HandTag),    // within the bound: kept
		snap("lonehand", at(12), "module=mail", HandTag), // past the bound, no night of mail after it: kept
		snap("mailnite", at(13), "module=mail"),          // a night of mail, but before it
		snap("pghand02", at(8.5), "module=pg", HandTag),  // past the bound, night001 is before it, but a night of
		snap("pgnight2", at(1), "module=pg"),             // pg after it exists: let go
		snap("retired1", at(30), "module=pg", "retired=data"),
		snap("otherhnd", at(11), "module=web", HandTag),          // a night of another module after it is no night of its own
		snap("ret-hand", at(11), "module=web", "retired=x"),      // a retired restore point is no night either
		snap("webnight", at(10.5), "module=web", "retired=data"), // nor this
		snap("tvhand01", at(10), "module=tv", HandTag),           // past the bound; only a partial night of tv after it
		snap("tvnight0", at(12), "module=tv"),                    // tv's good night, before it
		snap("tvpartl1", at(1), "module=tv"),                     // a partial night: a snapshot, and NOT backed up
	}
	goodNights := map[string]string{"pg": "pgnight2", "mail": "mailnite", "tv": "tvnight0"}
	got := handLetGo(snaps, goodNights, now, 7)
	var ids []string
	for _, s := range got {
		ids = append(ids, s.ShortID)
	}
	slices.Sort(ids)
	if want := []string{"oldhand1", "pghand02"}; !slices.Equal(ids, want) {
		t.Errorf("let go %v, want %v", ids, want)
	}
}

// The thinning forgets a hand-taken restore point past its bound by its id, and records which.
func TestTheThinningForgetsAnExpiredHandTakenRestorePointByItsId(t *testing.T) {
	now := time.Date(2026, 10, 20, 3, 0, 0, 0, time.UTC)
	h := &handRestic{made: "bbbbbbbb11", snaps: []Snapshot{
		{ID: "aaaaaaaa11", ShortID: "aaaaaaaa", Time: now.AddDate(0, 0, -9), Tags: []string{"module=pg", HandTag}},
		{ID: "bbbbbbbb11", ShortID: "bbbbbbbb", Time: now, Tags: []string{"module=pg"}},
	}}
	b := &Backups{Where: placed(t, "# pg\npath /\n"), Run: h.run, Now: func() time.Time { return now }, Say: quiet}
	_, err := b.BackUp(context.Background(), "")
	must(t, err)
	forgets := h.with("restic forget")
	if len(forgets) != 2 || forgets[0] != "restic forget --retry-lock 10m0s aaaaaaaa11" || !strings.HasPrefix(forgets[1], "restic forget --prune") {
		t.Fatalf("thinned with %q, want the expired one forgotten by id, then the rotation", forgets)
	}
	th := b.LastRun().Thinned
	if th == nil || !th.OK || !slices.Equal(th.LetGo, []string{"aaaaaaaa"}) {
		t.Errorf("recorded %+v, want aaaaaaaa let go", th)
	}
}

// When the restore points cannot be listed, nothing is thinned: a hand-taken one is never lost to a list
// that did not read.
func TestARestorePointListThatFailsThinsNothing(t *testing.T) {
	var forgot []string
	run := func(_ context.Context, name string, args ...string) (string, error) {
		if name != "restic" {
			return "", nil
		}
		switch args[5] {
		case "backup":
			return "{\"message_type\":\"summary\",\"snapshot_id\":\"abcdef0123456789\"}\n", nil
		case "snapshots":
			return "", errors.New("Fatal: repository is already locked")
		case "forget":
			forgot = append(forgot, strings.Join(args[5:], " "))
		}
		return "", nil
	}
	b := &Backups{Where: placed(t, "# pg\npath /\n"), Run: run, Now: time.Now, Say: quiet}
	_, err := b.BackUp(context.Background(), "")
	must(t, err)
	if len(forgot) != 0 {
		t.Errorf("thinned with %q", forgot)
	}
	if th := b.LastRun().Thinned; th == nil || th.OK || !strings.Contains(th.Error, "already locked") {
		t.Errorf("recorded %+v, want the failure", th)
	}
}

// backed-up says each hand-taken restore point and until when it is kept.
func TestBackedUpSaysWhichRestorePointsAreKeptAsHandTakenAndUntilWhen(t *testing.T) {
	now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC)
	early, late := now.Add(-10*time.Hour), now.Add(-time.Hour)
	h := &handRestic{snaps: []Snapshot{
		{ID: "aaaaaaaa11", ShortID: "aaaaaaaa", Time: early, Tags: []string{"module=pg", HandTag}},
		{ID: "bbbbbbbb11", ShortID: "bbbbbbbb", Time: now.Add(-5 * time.Hour), Tags: []string{"module=pg"}},
		{ID: "cccccccc11", ShortID: "cccccccc", Time: late, Tags: []string{"module=pg", HandTag}},
	}}
	b := &Backups{Where: placed(t, "# pg\npath /\n"), Run: h.run, Now: func() time.Time { return now }, Say: quiet}
	b.recordGoodNight("pg", "bbbbbbbb")
	listed, err := b.BackedUp(context.Background(), "pg")
	must(t, err)
	if len(listed) != 1 || len(listed[0].HandTaken) != 2 {
		t.Fatalf("listed %+v, want pg with two hand-taken restore points", listed)
	}
	first, second := listed[0].HandTaken[0], listed[0].HandTaken[1]
	if first.Snapshot != "aaaaaaaa" || !first.KeptAtLeastUntil.Equal(early.AddDate(0, 0, 7)) || first.WaitsForANight {
		t.Errorf("the first: %+v, want kept until 7 days after it, a night of pg after it already there", first)
	}
	if second.Snapshot != "cccccccc" || !second.KeptAtLeastUntil.Equal(late.AddDate(0, 0, 7)) || !second.WaitsForANight {
		t.Errorf("the second: %+v, want kept at least 7 days and until a later night", second)
	}
	if !strings.Contains(second.Kept, "a later good night of pg") || !strings.Contains(first.Kept, early.AddDate(0, 0, 7).Format(time.RFC3339)) {
		t.Errorf("said %q and %q", first.Kept, second.Kept)
	}
}

// The real thing, where restic is installed: the issue as it was seen — a backup taken by hand, then a night
// the same day — and the hand-taken restore point is still there; a week and a day later, with a later night
// of its module, the thinning lets it go.
func TestWithTheRealResticAHandTakenBackupOutlivesTheNightTheSameDay(t *testing.T) {
	if _, err := exec.LookPath("restic"); err != nil {
		t.Skip("restic is not installed")
	}
	root := t.TempDir()
	data := filepath.Join(root, "data")
	must(t, os.Mkdir(data, 0o700))
	must(t, os.WriteFile(filepath.Join(data, "library"), []byte("before the migration\n"), 0o600))
	where := placed(t, "# plex\npath "+data+"\n")
	// A machine whose counts are full, as ace's plex was: restic keeps the oldest restore point of a period not
	// yet filled, which would hide the issue on a fresh repository.
	conf, err := os.ReadFile(filepath.Join(where.State, RetentionFile))
	must(t, err)
	must(t, os.WriteFile(filepath.Join(where.State, RetentionFile),
		[]byte(strings.Replace(string(conf), "daily=14\nweekly=8\nmonthly=6", "daily=1\nweekly=1\nmonthly=1", 1)), 0o600))
	// As in the other real test: the directories above the test's own are said to be root's, so the restore
	// beside is what is tested.
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		if name == "stat" && len(args) > 0 && strings.HasPrefix(root, strings.TrimSuffix(args[len(args)-1], "/")+"/") {
			return "0 755 directory\n", nil
		}
		out, err := exec.CommandContext(ctx, name, args...).Output()
		if ee, ok := err.(*exec.ExitError); ok {
			return string(out), errors.New(string(ee.Stderr))
		}
		return string(out), err
	}
	clock := time.Now()
	b := &Backups{Where: where, Run: run, Now: func() time.Time { return clock }, Say: quiet}
	ctx := context.Background()

	hand, err := b.BackUpNow(ctx, "plex")
	must(t, err)
	must(t, os.WriteFile(filepath.Join(data, "library"), []byte("the damage\n"), 0o600))
	night, err := b.BackUp(ctx, "")
	must(t, err)
	if !hand["plex"].OK || !night["plex"].OK {
		t.Fatalf("hand %+v, night %+v", hand, night)
	}
	listed, err := b.BackedUp(ctx, "plex")
	must(t, err)
	if listed[0].RestorePoints != 2 {
		t.Fatalf("%d restore points after the night, want the hand-taken one kept beside the night's", listed[0].RestorePoints)
	}
	if len(listed[0].HandTaken) != 1 || listed[0].HandTaken[0].Snapshot != hand["plex"].Snapshot {
		t.Fatalf("hand-taken %+v, want %s", listed[0].HandTaken, hand["plex"].Snapshot)
	}
	r, err := b.Restore(ctx, "plex", hand["plex"].Snapshot, "")
	must(t, err)
	if raw, _ := os.ReadFile(filepath.Join(r.Restored[0], "library")); string(raw) != "before the migration\n" {
		t.Errorf("the hand-taken restore point holds %q", raw)
	}

	// Eight days on, a night of plex later than it exists: the thinning lets it go.
	clock = clock.AddDate(0, 0, 8)
	_, err = b.BackUp(ctx, "")
	must(t, err)
	listed, err = b.BackedUp(ctx, "plex")
	must(t, err)
	if len(listed[0].HandTaken) != 0 {
		t.Errorf("hand-taken %+v eight days on, want it let go", listed[0].HandTaken)
	}
	if th := b.LastRun().Thinned; th == nil || !slices.Equal(th.LetGo, []string{hand["plex"].Snapshot}) {
		t.Errorf("recorded %+v, want the hand-taken restore point let go", th)
	}
}

// A partial night — restic exits 3, a source file could not be read, and still writes a snapshot — is recorded
// as NOT backed up, and is no later night: a restore point taken by hand past its days is kept beside it.
func TestAPartialNightIsNoLaterNightForARestorePointTakenByHand(t *testing.T) {
	now := time.Date(2026, 10, 20, 3, 0, 0, 0, time.UTC)
	h := &handRestic{partial: true, snaps: []Snapshot{
		{ID: "nnnnnnnn11", ShortID: "nnnnnnnn", Time: now.AddDate(0, 0, -12), Tags: []string{"module=pg"}},
		{ID: "aaaaaaaa11", ShortID: "aaaaaaaa", Time: now.AddDate(0, 0, -9), Tags: []string{"module=pg", HandTag}},
		// What the partial night below left: a snapshot of pg alone.
		{ID: "pppppppp11", ShortID: "pppppppp", Time: now, Tags: []string{"module=pg"}},
	}}
	b := &Backups{Where: placed(t, "# pg\npath /\n"), Run: h.run, Now: func() time.Time { return now }, Say: quiet}
	b.recordGoodNight("pg", "nnnnnnnn")
	outcome, err := b.BackUp(context.Background(), "")
	must(t, err)
	if outcome["pg"].OK {
		t.Fatalf("the partial night is recorded good: %+v", outcome["pg"])
	}
	for _, f := range h.with("restic forget") {
		if strings.Contains(f, "aaaaaaaa") {
			t.Errorf("let go the restore point taken by hand on a partial night: %q", f)
		}
	}
	listed, err := b.BackedUp(context.Background(), "pg")
	must(t, err)
	if len(listed[0].HandTaken) != 1 || !listed[0].HandTaken[0].WaitsForANight {
		t.Errorf("hand-taken %+v, want it still waiting for a good night", listed[0].HandTaken)
	}
}

// When the counts' thinning fails after restore points taken by hand were let go, those are said gone: never
// "nothing was forgotten".
func TestAThinningThatFailsAfterLettingGoSaysWhatItLetGo(t *testing.T) {
	now := time.Date(2026, 10, 20, 3, 0, 0, 0, time.UTC)
	h := &handRestic{made: "bbbbbbbb11", snaps: []Snapshot{
		{ID: "aaaaaaaa11", ShortID: "aaaaaaaa", Time: now.AddDate(0, 0, -9), Tags: []string{"module=pg", HandTag}},
		{ID: "bbbbbbbb11", ShortID: "bbbbbbbb", Time: now, Tags: []string{"module=pg"}},
	}}
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		if name == "restic" && args[5] == "forget" && len(args) > 6 && args[6] == "--prune" {
			return "", errors.New("Fatal: repository is already locked")
		}
		return h.run(ctx, name, args...)
	}
	b := &Backups{Where: placed(t, "# pg\npath /\n"), Run: run, Now: func() time.Time { return now }, Say: quiet}
	_, err := b.BackUp(context.Background(), "")
	must(t, err)
	if th := b.LastRun().Thinned; th == nil || th.OK || !slices.Equal(th.LetGo, []string{"aaaaaaaa"}) {
		t.Fatalf("recorded %+v, want the failure and aaaaaaaa let go", th)
	}
	why := b.Health(context.Background()).Why
	if strings.Contains(why, "Nothing was forgotten") || !strings.Contains(why, "let go aaaaaaaa") {
		t.Errorf("said %q, want it to say aaaaaaaa was let go", why)
	}
}

// A backup asked for now that cannot start leaves its failure as the module's last night, so the mesh asking
// for it before a delivery reads the backup's own words from backed-up, not silence (novox/hq issue 1067).
func TestABackupAskedForNowThatCannotStartIsRecordedAsThatModulesLastNight(t *testing.T) {
	h := &handRestic{}
	asked := time.Date(2026, 10, 17, 22, 50, 0, 0, time.UTC)
	b := &Backups{Where: placed(t, "# pg\npath /\n"), Run: h.run, Now: func() time.Time { return asked }, Say: quiet}
	must(t, os.Mkdir(filepath.Join(b.Where.State, nightLock), 0o700)) // a directory cannot be opened read-write: the lock cannot be taken
	if _, err := b.BackUpNow(context.Background(), "pg"); err == nil {
		t.Fatal("a backup started without its lock")
	}
	night, ok := b.Nights()["pg"]
	if !ok || night.OK || night.Error == "" || !night.At.Equal(asked) {
		t.Fatalf("pg's last night is %+v, want a failure at %s with the reason", night, asked)
	}
	if !strings.Contains(night.Error, nightLock) {
		t.Errorf("the reason is %q, want it to name the lock", night.Error)
	}
	// Asked of a module that declares nothing here, once the lock can be taken: recorded too, with the reason.
	must(t, os.Remove(filepath.Join(b.Where.State, nightLock)))
	if _, err := b.BackUpNow(context.Background(), "mail"); err == nil {
		t.Fatal("a backup of a module that declares nothing started")
	}
	if n := b.Nights()["mail"]; n.OK || !strings.Contains(n.Error, "declares nothing") {
		t.Errorf("mail's last night is %+v, want a failure saying it declares nothing", n)
	}
}