package main // A backup taken by hand — `node-backup.now`, asked by a person or by the mesh before a module moves (novox/hq // ADR 0233) — is kept by the thinning as its own restore point: until a later good night of the same module // exists, and for the machine's keep-hand-taken-days in any case (novox/hq issue 453: one taken at 02:35 was // forgotten by the night at 03:01, as keep-daily keeps only the newest restore point of each day). import ( "context" "encoding/json" "errors" "os" "os/exec" "path/filepath" "slices" "strings" "testing" "time" ) // handRestic is a fake restic that remembers each command and lists the snapshots it is given. A backup makes // the snapshot made (abcdef01 when empty), and fails as restic's exit 3 when partial. type handRestic struct { calls []string snaps []Snapshot made string partial bool } func (h *handRestic) run(_ context.Context, name string, args ...string) (string, error) { if name != "restic" { return "", nil } line := "restic " + strings.Join(args[5:], " ") h.calls = append(h.calls, line) switch args[5] { case "backup": if h.partial { return "", errors.New("Warning: at least one source file could not be read") } made := h.made if made == "" { made = "abcdef0123456789" } return "{\"message_type\":\"summary\",\"snapshot_id\":\"" + made + "\"}\n", nil case "snapshots": raw, _ := json.Marshal(h.snaps) return string(raw), nil } return "", nil } func (h *handRestic) with(prefix string) []string { var out []string for _, c := range h.calls { if strings.HasPrefix(c, prefix) { out = append(out, c) } } return out } func TestAHandTakenBackupIsTaggedAndTheThinningKeepsThatTag(t *testing.T) { h := &handRestic{} b := &Backups{Where: placed(t, "# pg\npath /\n"), Run: h.run, Now: func() time.Time { return time.Date(2026, 10, 11, 2, 35, 0, 0, time.UTC) }, Say: quiet} _, err := b.BackUpNow(context.Background(), "pg") must(t, err) if got := h.with("restic backup"); len(got) != 1 || got[0] != "restic backup --json --tag module=pg --tag hand /" { t.Errorf("backed up by hand with %q, want it tagged hand", got) } _, err = b.BackUp(context.Background(), "") must(t, err) if got := h.with("restic backup"); len(got) != 2 || got[1] != "restic backup --json --tag module=pg /" { t.Errorf("the night backed up with %q, want it tagged with its module alone", got) } want := "restic forget --prune --group-by host,tags --keep-daily 14 --keep-weekly 8 --keep-monthly 6 --keep-tag hand" for _, f := range h.with("restic forget") { if f != want { t.Errorf("thinned with %q, want %q", f, want) } // No rule that would keep every night as well: the hand tag is the only thing kept beyond the counts. if strings.Contains(f, "--keep-within") || strings.Contains(f, "--keep-last") { t.Errorf("thinned with %q, a rule that keeps nights too", f) } } if len(h.with("restic forget")) != 2 { t.Errorf("thinned %d times, want after each of the two backups", len(h.with("restic forget"))) } } func TestAHandTakenRestorePointIsLetGoOnlyPastItsBoundAndAfterALaterNight(t *testing.T) { now := time.Date(2026, 10, 20, 3, 0, 0, 0, time.UTC) at := func(days float64) time.Time { return now.Add(-time.Duration(days * 24 * float64(time.Hour))) } snap := func(id string, when time.Time, tags ...string) Snapshot { return Snapshot{ID: id + "0000000000", ShortID: id, Time: when, Tags: tags} } snaps := []Snapshot{ snap("oldhand1", at(10), "module=pg", HandTag), // past the bound, a night after it: let go snap("night001", at(9), "module=pg"), // the night after it snap("freshhnd", at(2), "module=pg", HandTag), // within the bound: kept snap("lonehand", at(12), "module=mail", HandTag), // past the bound, no night of mail after it: kept snap("mailnite", at(13), "module=mail"), // a night of mail, but before it snap("pghand02", at(8.5), "module=pg", HandTag), // past the bound, night001 is before it, but a night of snap("pgnight2", at(1), "module=pg"), // pg after it exists: let go snap("retired1", at(30), "module=pg", "retired=data"), snap("otherhnd", at(11), "module=web", HandTag), // a night of another module after it is no night of its own snap("ret-hand", at(11), "module=web", "retired=x"), // a retired restore point is no night either snap("webnight", at(10.5), "module=web", "retired=data"), // nor this snap("tvhand01", at(10), "module=tv", HandTag), // past the bound; only a partial night of tv after it snap("tvnight0", at(12), "module=tv"), // tv's good night, before it snap("tvpartl1", at(1), "module=tv"), // a partial night: a snapshot, and NOT backed up } goodNights := map[string]string{"pg": "pgnight2", "mail": "mailnite", "tv": "tvnight0"} got := handLetGo(snaps, goodNights, now, 7) var ids []string for _, s := range got { ids = append(ids, s.ShortID) } slices.Sort(ids) if want := []string{"oldhand1", "pghand02"}; !slices.Equal(ids, want) { t.Errorf("let go %v, want %v", ids, want) } } // The thinning forgets a hand-taken restore point past its bound by its id, and records which. func TestTheThinningForgetsAnExpiredHandTakenRestorePointByItsId(t *testing.T) { now := time.Date(2026, 10, 20, 3, 0, 0, 0, time.UTC) h := &handRestic{made: "bbbbbbbb11", snaps: []Snapshot{ {ID: "aaaaaaaa11", ShortID: "aaaaaaaa", Time: now.AddDate(0, 0, -9), Tags: []string{"module=pg", HandTag}}, {ID: "bbbbbbbb11", ShortID: "bbbbbbbb", Time: now, Tags: []string{"module=pg"}}, }} b := &Backups{Where: placed(t, "# pg\npath /\n"), Run: h.run, Now: func() time.Time { return now }, Say: quiet} _, err := b.BackUp(context.Background(), "") must(t, err) forgets := h.with("restic forget") if len(forgets) != 2 || forgets[0] != "restic forget aaaaaaaa11" || !strings.HasPrefix(forgets[1], "restic forget --prune") { t.Fatalf("thinned with %q, want the expired one forgotten by id, then the rotation", forgets) } th := b.LastRun().Thinned if th == nil || !th.OK || !slices.Equal(th.LetGo, []string{"aaaaaaaa"}) { t.Errorf("recorded %+v, want aaaaaaaa let go", th) } } // When the restore points cannot be listed, nothing is thinned: a hand-taken one is never lost to a list // that did not read. func TestARestorePointListThatFailsThinsNothing(t *testing.T) { var forgot []string run := func(_ context.Context, name string, args ...string) (string, error) { if name != "restic" { return "", nil } switch args[5] { case "backup": return "{\"message_type\":\"summary\",\"snapshot_id\":\"abcdef0123456789\"}\n", nil case "snapshots": return "", errors.New("Fatal: repository is already locked") case "forget": forgot = append(forgot, strings.Join(args[5:], " ")) } return "", nil } b := &Backups{Where: placed(t, "# pg\npath /\n"), Run: run, Now: time.Now, Say: quiet} _, err := b.BackUp(context.Background(), "") must(t, err) if len(forgot) != 0 { t.Errorf("thinned with %q", forgot) } if th := b.LastRun().Thinned; th == nil || th.OK || !strings.Contains(th.Error, "already locked") { t.Errorf("recorded %+v, want the failure", th) } } // backed-up says each hand-taken restore point and until when it is kept. func TestBackedUpSaysWhichRestorePointsAreKeptAsHandTakenAndUntilWhen(t *testing.T) { now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC) early, late := now.Add(-10*time.Hour), now.Add(-time.Hour) h := &handRestic{snaps: []Snapshot{ {ID: "aaaaaaaa11", ShortID: "aaaaaaaa", Time: early, Tags: []string{"module=pg", HandTag}}, {ID: "bbbbbbbb11", ShortID: "bbbbbbbb", Time: now.Add(-5 * time.Hour), Tags: []string{"module=pg"}}, {ID: "cccccccc11", ShortID: "cccccccc", Time: late, Tags: []string{"module=pg", HandTag}}, }} b := &Backups{Where: placed(t, "# pg\npath /\n"), Run: h.run, Now: func() time.Time { return now }, Say: quiet} b.recordGoodNight("pg", "bbbbbbbb") listed, err := b.BackedUp(context.Background(), "pg") must(t, err) if len(listed) != 1 || len(listed[0].HandTaken) != 2 { t.Fatalf("listed %+v, want pg with two hand-taken restore points", listed) } first, second := listed[0].HandTaken[0], listed[0].HandTaken[1] if first.Snapshot != "aaaaaaaa" || !first.KeptAtLeastUntil.Equal(early.AddDate(0, 0, 7)) || first.WaitsForANight { t.Errorf("the first: %+v, want kept until 7 days after it, a night of pg after it already there", first) } if second.Snapshot != "cccccccc" || !second.KeptAtLeastUntil.Equal(late.AddDate(0, 0, 7)) || !second.WaitsForANight { t.Errorf("the second: %+v, want kept at least 7 days and until a later night", second) } if !strings.Contains(second.Kept, "a later good night of pg") || !strings.Contains(first.Kept, early.AddDate(0, 0, 7).Format(time.RFC3339)) { t.Errorf("said %q and %q", first.Kept, second.Kept) } } // The real thing, where restic is installed: the issue as it was seen — a backup taken by hand, then a night // the same day — and the hand-taken restore point is still there; a week and a day later, with a later night // of its module, the thinning lets it go. func TestWithTheRealResticAHandTakenBackupOutlivesTheNightTheSameDay(t *testing.T) { if _, err := exec.LookPath("restic"); err != nil { t.Skip("restic is not installed") } root := t.TempDir() data := filepath.Join(root, "data") must(t, os.Mkdir(data, 0o700)) must(t, os.WriteFile(filepath.Join(data, "library"), []byte("before the migration\n"), 0o600)) where := placed(t, "# plex\npath "+data+"\n") // A machine whose counts are full, as ace's plex was: restic keeps the oldest restore point of a period not // yet filled, which would hide the issue on a fresh repository. conf, err := os.ReadFile(filepath.Join(where.State, RetentionFile)) must(t, err) must(t, os.WriteFile(filepath.Join(where.State, RetentionFile), []byte(strings.Replace(string(conf), "daily=14\nweekly=8\nmonthly=6", "daily=1\nweekly=1\nmonthly=1", 1)), 0o600)) // As in the other real test: the directories above the test's own are said to be root's, so the restore // beside is what is tested. run := func(ctx context.Context, name string, args ...string) (string, error) { if name == "stat" && len(args) > 0 && strings.HasPrefix(root, strings.TrimSuffix(args[len(args)-1], "/")+"/") { return "0 755 directory\n", nil } out, err := exec.CommandContext(ctx, name, args...).Output() if ee, ok := err.(*exec.ExitError); ok { return string(out), errors.New(string(ee.Stderr)) } return string(out), err } clock := time.Now() b := &Backups{Where: where, Run: run, Now: func() time.Time { return clock }, Say: quiet} ctx := context.Background() hand, err := b.BackUpNow(ctx, "plex") must(t, err) must(t, os.WriteFile(filepath.Join(data, "library"), []byte("the damage\n"), 0o600)) night, err := b.BackUp(ctx, "") must(t, err) if !hand["plex"].OK || !night["plex"].OK { t.Fatalf("hand %+v, night %+v", hand, night) } listed, err := b.BackedUp(ctx, "plex") must(t, err) if listed[0].RestorePoints != 2 { t.Fatalf("%d restore points after the night, want the hand-taken one kept beside the night's", listed[0].RestorePoints) } if len(listed[0].HandTaken) != 1 || listed[0].HandTaken[0].Snapshot != hand["plex"].Snapshot { t.Fatalf("hand-taken %+v, want %s", listed[0].HandTaken, hand["plex"].Snapshot) } r, err := b.Restore(ctx, "plex", hand["plex"].Snapshot, "") must(t, err) if raw, _ := os.ReadFile(filepath.Join(r.Restored[0], "library")); string(raw) != "before the migration\n" { t.Errorf("the hand-taken restore point holds %q", raw) } // Eight days on, a night of plex later than it exists: the thinning lets it go. clock = clock.AddDate(0, 0, 8) _, err = b.BackUp(ctx, "") must(t, err) listed, err = b.BackedUp(ctx, "plex") must(t, err) if len(listed[0].HandTaken) != 0 { t.Errorf("hand-taken %+v eight days on, want it let go", listed[0].HandTaken) } if th := b.LastRun().Thinned; th == nil || !slices.Equal(th.LetGo, []string{hand["plex"].Snapshot}) { t.Errorf("recorded %+v, want the hand-taken restore point let go", th) } } // A partial night — restic exits 3, a source file could not be read, and still writes a snapshot — is recorded // as NOT backed up, and is no later night: a restore point taken by hand past its days is kept beside it. func TestAPartialNightIsNoLaterNightForARestorePointTakenByHand(t *testing.T) { now := time.Date(2026, 10, 20, 3, 0, 0, 0, time.UTC) h := &handRestic{partial: true, snaps: []Snapshot{ {ID: "nnnnnnnn11", ShortID: "nnnnnnnn", Time: now.AddDate(0, 0, -12), Tags: []string{"module=pg"}}, {ID: "aaaaaaaa11", ShortID: "aaaaaaaa", Time: now.AddDate(0, 0, -9), Tags: []string{"module=pg", HandTag}}, // What the partial night below left: a snapshot of pg alone. {ID: "pppppppp11", ShortID: "pppppppp", Time: now, Tags: []string{"module=pg"}}, }} b := &Backups{Where: placed(t, "# pg\npath /\n"), Run: h.run, Now: func() time.Time { return now }, Say: quiet} b.recordGoodNight("pg", "nnnnnnnn") outcome, err := b.BackUp(context.Background(), "") must(t, err) if outcome["pg"].OK { t.Fatalf("the partial night is recorded good: %+v", outcome["pg"]) } for _, f := range h.with("restic forget") { if strings.Contains(f, "aaaaaaaa") { t.Errorf("let go the restore point taken by hand on a partial night: %q", f) } } listed, err := b.BackedUp(context.Background(), "pg") must(t, err) if len(listed[0].HandTaken) != 1 || !listed[0].HandTaken[0].WaitsForANight { t.Errorf("hand-taken %+v, want it still waiting for a good night", listed[0].HandTaken) } } // When the counts' thinning fails after restore points taken by hand were let go, those are said gone: never // "nothing was forgotten". func TestAThinningThatFailsAfterLettingGoSaysWhatItLetGo(t *testing.T) { now := time.Date(2026, 10, 20, 3, 0, 0, 0, time.UTC) h := &handRestic{made: "bbbbbbbb11", snaps: []Snapshot{ {ID: "aaaaaaaa11", ShortID: "aaaaaaaa", Time: now.AddDate(0, 0, -9), Tags: []string{"module=pg", HandTag}}, {ID: "bbbbbbbb11", ShortID: "bbbbbbbb", Time: now, Tags: []string{"module=pg"}}, }} run := func(ctx context.Context, name string, args ...string) (string, error) { if name == "restic" && args[5] == "forget" && len(args) > 6 && args[6] == "--prune" { return "", errors.New("Fatal: repository is already locked") } return h.run(ctx, name, args...) } b := &Backups{Where: placed(t, "# pg\npath /\n"), Run: run, Now: func() time.Time { return now }, Say: quiet} _, err := b.BackUp(context.Background(), "") must(t, err) if th := b.LastRun().Thinned; th == nil || th.OK || !slices.Equal(th.LetGo, []string{"aaaaaaaa"}) { t.Fatalf("recorded %+v, want the failure and aaaaaaaa let go", th) } why := b.Health(context.Background()).Why if strings.Contains(why, "Nothing was forgotten") || !strings.Contains(why, "let go aaaaaaaa") { t.Errorf("said %q, want it to say aaaaaaaa was let go", why) } }