diff --git a/modules/restic/cmd/restic-backups/backups.go b/modules/restic/cmd/restic-backups/backups.go index aba6086e..460ea7bd 100644 --- a/modules/restic/cmd/restic-backups/backups.go +++ b/modules/restic/cmd/restic-backups/backups.go @@ -8,7 +8,7 @@ // path a directory the module's snapshot keeps // // Each module gets one snapshot a night, tagged with its name, so a module is listed, kept and -// restored on its own. Everything lands in one repository on the machine — deduplicated, so every +// restored on its own; one taken by hand is tagged as such too, and kept apart (hand.go). Everything lands in one repository on the machine — deduplicated, so every // night is a complete restore point and only what changed costs space — and is thinned by the machine's // settings keep-daily, keep-weekly and keep-monthly (retention.go). Against mistakes, not disasters: // nothing leaves the machine. How the last night and thinning went is the module's health (health.go). @@ -277,7 +277,7 @@ func (b *Backups) ensureRepository(ctx context.Context) error { // one is one module's night: its run lines, then one snapshot of its paths. A failure is that // module's alone. -func (b *Backups) one(ctx context.Context, d Declared) Night { +func (b *Backups) one(ctx context.Context, d Declared, hand bool) Night { n := Night{At: b.Now().UTC()} fail := func(err error) Night { n.Error = err.Error() @@ -301,7 +301,11 @@ func (b *Backups) one(ctx context.Context, d Declared) Night { if len(missing) > 0 { return fail(fmt.Errorf("%s does not exist", strings.Join(missing, ", "))) } - out, err := b.restic(ctx, append([]string{"backup", "--json", "--tag", tagOf(d.Module)}, d.Paths...)...) + args := []string{"backup", "--json", "--tag", tagOf(d.Module)} + if hand { + args = append(args, "--tag", HandTag) + } + out, err := b.restic(ctx, append(args, d.Paths...)...) if err != nil { return fail(err) } @@ -332,6 +336,11 @@ func snapshotOf(out string) string { // BackUp is a night: every module, or one, then the rotation. It answers each module's outcome, and // keeps how the night as a whole and its thinning went for the health (novox/hq issue 454). func (b *Backups) BackUp(ctx context.Context, only string) (map[string]Night, error) { + return b.backUp(ctx, only, false) +} + +// backUp is a night, or a backup taken by hand (hand.go): the same but for the restore points' tag. +func (b *Backups) backUp(ctx context.Context, only string, hand bool) (map[string]Night, error) { run := b.LastRun() run.At = b.Now().UTC() failed := func(err error) (map[string]Night, error) { @@ -368,7 +377,7 @@ func (b *Backups) BackUp(ctx context.Context, only string) (map[string]Night, er } outcome := map[string]Night{} for _, d := range chosen { - outcome[d.Module] = b.one(ctx, d) + outcome[d.Module] = b.one(ctx, d, hand) b.record(d.Module, outcome[d.Module]) } // A retention that does not read thins nothing: keeping too much costs space, thinning by a value @@ -378,18 +387,42 @@ func (b *Backups) BackUp(ctx context.Context, only string) (map[string]Night, er if err != nil { thinned.Refused = err.Error() b.Say("the retention does not read, so nothing is thinned and every restore point is kept: %v", err) - } else if _, err := b.restic(ctx, "forget", "--prune", "--group-by", "host,tags", - "--keep-daily", fmt.Sprint(keep.Daily), "--keep-weekly", fmt.Sprint(keep.Weekly), "--keep-monthly", fmt.Sprint(keep.Monthly)); err != nil { + } else if letGo, err := b.thin(ctx, keep); err != nil { thinned.Kept, thinned.Error = &keep, err.Error() b.Say("thinning the restore points failed, and every one is kept: %v", err) } else { - thinned.Kept, thinned.OK = &keep, true + thinned.Kept, thinned.OK, thinned.LetGo = &keep, true, letGo } run.Failed, run.Thinned = "", &thinned b.keepRun(run) return outcome, nil } +// thin forgets the restore points taken by hand that are past their days and have a later night of their +// module, by id, then thins the rest by the counts, keeping every one still tagged as taken by hand (novox/hq +// issue 453). It answers the short ids it let go. A list that does not read thins nothing: a restore point +// taken by hand is never lost to a list nobody could read. +func (b *Backups) thin(ctx context.Context, keep Retention) ([]string, error) { + snaps, err := b.Snapshots(ctx, "") + if err != nil { + return nil, fmt.Errorf("the restore points could not be listed: %w", err) + } + var ids, letGo []string + for _, s := range handLetGo(snaps, b.Now(), keep.Hand) { + ids, letGo = append(ids, s.ID), append(letGo, s.ShortID) + } + if len(ids) > 0 { + if _, err := b.restic(ctx, append([]string{"forget"}, ids...)...); err != nil { + return nil, err + } + b.Say("let go the restore points taken by hand past %d days with a later night: %s", keep.Hand, strings.Join(letGo, ", ")) + } + _, err = b.restic(ctx, "forget", "--prune", "--group-by", "host,tags", + "--keep-daily", fmt.Sprint(keep.Daily), "--keep-weekly", fmt.Sprint(keep.Weekly), "--keep-monthly", fmt.Sprint(keep.Monthly), + "--keep-tag", HandTag) + return letGo, err +} + // Run is how the last night, or backup asked for now, went as a whole: whether it ran at all, and its // thinning. Each module's own outcome is its Night. type Run struct { @@ -406,6 +439,8 @@ type Thinning struct { Kept *Retention `json:"kept,omitempty"` Refused string `json:"refused,omitempty"` Error string `json:"error,omitempty"` + // LetGo is the restore points taken by hand it forgot, past their days and with a later night (hand.go). + LetGo []string `json:"letGo,omitempty"` } func (b *Backups) runFile() string { return filepath.Join(b.Where.State, "run.json") } @@ -469,6 +504,8 @@ type ModuleBackups struct { // Data is every item the module declares, measured, with its newest good backup (novox/hq ADR // 0233): what the controller's self-check reads. Data []ItemReport `json:"data"` + // HandTaken is its restore points taken by hand, each with until when it is kept (novox/hq issue 453). + HandTaken []HandKept `json:"handTaken"` } // BackedUp is what is backed up here: each module, what it declared, its last night and its restore @@ -496,7 +533,8 @@ func (b *Backups) BackedUp(ctx context.Context, module string) ([]ModuleBackups, } listed[d.Module] = true m := ModuleBackups{Module: d.Module, Runs: len(d.Runs), Paths: d.Paths, Retention: inForce, Thinned: thinned, - Data: itemReports(items[d.Module], measured[d.Module], d.Paths, good[d.Module])} + Data: itemReports(items[d.Module], measured[d.Module], d.Paths, good[d.Module]), + HandTaken: handKept(d.Module, snaps, inForce)} if n, ok := nights[d.Module]; ok { m.LastNight = &n } @@ -514,7 +552,7 @@ func (b *Backups) BackedUp(ctx context.Context, module string) ([]ModuleBackups, if listed[name] || (module != "" && name != module) { continue } - out = append(out, ModuleBackups{Module: name, Paths: []string{}, Data: itemReports(its, measured[name], nil, time.Time{})}) + out = append(out, ModuleBackups{Module: name, Paths: []string{}, HandTaken: []HandKept{}, Data: itemReports(its, measured[name], nil, time.Time{})}) } return out, nil } diff --git a/modules/restic/cmd/restic-backups/backups_test.go b/modules/restic/cmd/restic-backups/backups_test.go index bf6ee69f..24930540 100644 --- a/modules/restic/cmd/restic-backups/backups_test.go +++ b/modules/restic/cmd/restic-backups/backups_test.go @@ -26,7 +26,7 @@ func placed(t *testing.T, declared string) Where { must(t, os.WriteFile(filepath.Join(dir, "backups.conf"), []byte(declared), 0o600)) must(t, os.WriteFile(filepath.Join(dir, "pw"), []byte("secret\n"), 0o600)) // The retention file every machine has, as the mesh writes it with the defaults (novox/hq issue 454). - must(t, os.WriteFile(filepath.Join(dir, RetentionFile), []byte("daily=14\nweekly=8\nmonthly=6\n"), 0o600)) + must(t, os.WriteFile(filepath.Join(dir, RetentionFile), []byte("daily=14\nweekly=8\nmonthly=6\nhand=7\n"), 0o600)) return Where{Declared: filepath.Join(dir, "backups.conf"), Repository: filepath.Join(dir, "repo"), PasswordFile: filepath.Join(dir, "pw"), State: dir} } @@ -107,7 +107,8 @@ func TestANightDumpsBeforeEachSnapshotAndOneFailingModuleFailsOnlyItself(t *test "restic backup --json --tag module=pg /", "sh -c fail-it", "restic backup --json --tag module=mail /", - "restic forget --prune --group-by host,tags --keep-daily 14 --keep-weekly 8 --keep-monthly 6", + "restic snapshots --json", + "restic forget --prune --group-by host,tags --keep-daily 14 --keep-weekly 8 --keep-monthly 6 --keep-tag hand", } if !reflect.DeepEqual(calls, want) { t.Errorf("ran\n%s\nwant\n%s", strings.Join(calls, "\n"), strings.Join(want, "\n")) diff --git a/modules/restic/cmd/restic-backups/hand.go b/modules/restic/cmd/restic-backups/hand.go new file mode 100644 index 00000000..d97b6294 --- /dev/null +++ b/modules/restic/cmd/restic-backups/hand.go @@ -0,0 +1,108 @@ +package main + +// A backup taken by hand (novox/hq issue 453): `node-backup.now`, asked by a person before a risky change or by +// the mesh before a module moves (ADR 0233, the bus's snapshot before its upgrade). The thinning after every +// backup keeps only the newest restore point of each day, so one taken at 02:35 was forgotten by the night at +// 03:01 — while the night's may already hold the damage it was taken against. +// +// So a backup taken by hand is tagged HandTag beside its module's tag, and is kept by restic's own --keep-tag: +// with the thinning grouping by tags, it is its own group too, never counted against the nights. It is kept +// until a later good night of its module exists, and for the machine's keep-hand-taken-days in any case; past +// both, the thinning forgets it by its id (handLetGo). The nights are thinned by their counts alone: no rule +// that keeps one keeps every night. + +import ( + "context" + "fmt" + "slices" + "strings" + "time" +) + +// HandTag marks a restore point taken by hand. +const HandTag = "hand" + +// BackUpNow is a backup asked for now, of every module or one: as a night, each restore point tagged HandTag. +func (b *Backups) BackUpNow(ctx context.Context, only string) (map[string]Night, error) { + return b.backUp(ctx, only, true) +} + +// isHand is whether a restore point was taken by hand. +func isHand(s Snapshot) bool { return slices.Contains(s.Tags, HandTag) } + +// moduleOf is the module a restore point is of; empty when it carries no module's tag. +func moduleOf(s Snapshot) string { + for _, t := range s.Tags { + if m, ok := strings.CutPrefix(t, "module="); ok && m != "" { + return m + } + } + return "" +} + +// isNight is whether a restore point is a night's: its module's tag and nothing else — neither taken by hand +// nor a retired item's last one. Restic writes a snapshot only for a backup that succeeded, so each is good. +func isNight(s Snapshot) bool { return len(s.Tags) == 1 && moduleOf(s) != "" } + +// latestNights is the time of each module's newest night. +func latestNights(snaps []Snapshot) map[string]time.Time { + latest := map[string]time.Time{} + for _, s := range snaps { + if m := moduleOf(s); isNight(s) && s.Time.After(latest[m]) { + latest[m] = s.Time + } + } + return latest +} + +// handLetGo is the restore points taken by hand that the thinning forgets: older than days, and with a night of +// the same module after them. Every other one taken by hand is kept. +func handLetGo(snaps []Snapshot, now time.Time, days int) []Snapshot { + latest := latestNights(snaps) + cutoff := now.AddDate(0, 0, -days) + var out []Snapshot + for _, s := range snaps { + if isHand(s) && s.Time.Before(cutoff) && latest[moduleOf(s)].After(s.Time) { + out = append(out, s) + } + } + return out +} + +// HandKept is one restore point taken by hand, as `backed-up` says it. +type HandKept struct { + Snapshot string `json:"snapshot"` + At time.Time `json:"at"` + // KeptAtLeastUntil is when its days are up; WaitsForANight that no later good night of its module exists + // yet, and it is kept until one does as well. The first thinning after both forgets it. + KeptAtLeastUntil time.Time `json:"keptAtLeastUntil"` + WaitsForANight bool `json:"waitsForANight"` + Kept string `json:"kept"` +} + +// handKept is a module's restore points taken by hand, oldest first, said with the retention in force. +func handKept(module string, snaps []Snapshot, keep InForce) []HandKept { + latest := latestNights(snaps) + out := []HandKept{} + for _, s := range snaps { + if !isHand(s) || moduleOf(s) != module { + continue + } + k := HandKept{Snapshot: s.ShortID, At: s.Time, WaitsForANight: !latest[module].After(s.Time)} + switch { + case keep.Retention == nil: + k.Kept = "kept: nothing is thinned while the retention is refused" + case k.WaitsForANight: + k.KeptAtLeastUntil = s.Time.AddDate(0, 0, keep.Hand) + k.Kept = fmt.Sprintf("kept as taken by hand until a later good night of %s exists, and at least until %s", + module, k.KeptAtLeastUntil.UTC().Format(time.RFC3339)) + default: + k.KeptAtLeastUntil = s.Time.AddDate(0, 0, keep.Hand) + k.Kept = fmt.Sprintf("kept as taken by hand until %s; the first thinning after forgets it", + k.KeptAtLeastUntil.UTC().Format(time.RFC3339)) + } + out = append(out, k) + } + slices.SortFunc(out, func(a, b HandKept) int { return a.At.Compare(b.At) }) + return out +} diff --git a/modules/restic/cmd/restic-backups/hand_test.go b/modules/restic/cmd/restic-backups/hand_test.go new file mode 100644 index 00000000..92d2390d --- /dev/null +++ b/modules/restic/cmd/restic-backups/hand_test.go @@ -0,0 +1,256 @@ +package main + +// A backup taken by hand — `node-backup.now`, asked by a person or by the mesh before a module moves (novox/hq +// ADR 0233) — is kept by the thinning as its own restore point: until a later good night of the same module +// exists, and for the machine's keep-hand-taken-days in any case (novox/hq issue 453: one taken at 02:35 was +// forgotten by the night at 03:01, as keep-daily keeps only the newest restore point of each day). + +import ( + "context" + "encoding/json" + "errors" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "testing" + "time" +) + +// handRestic is a fake restic that remembers each command and lists the snapshots it is given. +type handRestic struct { + calls []string + snaps []Snapshot +} + +func (h *handRestic) run(_ context.Context, name string, args ...string) (string, error) { + if name != "restic" { + return "", nil + } + line := "restic " + strings.Join(args[5:], " ") + h.calls = append(h.calls, line) + switch args[5] { + case "backup": + return "{\"message_type\":\"summary\",\"snapshot_id\":\"abcdef0123456789\"}\n", nil + case "snapshots": + raw, _ := json.Marshal(h.snaps) + return string(raw), nil + } + return "", nil +} + +func (h *handRestic) with(prefix string) []string { + var out []string + for _, c := range h.calls { + if strings.HasPrefix(c, prefix) { + out = append(out, c) + } + } + return out +} + +func TestAHandTakenBackupIsTaggedAndTheThinningKeepsThatTag(t *testing.T) { + h := &handRestic{} + b := &Backups{Where: placed(t, "# pg\npath /\n"), Run: h.run, + Now: func() time.Time { return time.Date(2026, 10, 11, 2, 35, 0, 0, time.UTC) }, Say: quiet} + _, err := b.BackUpNow(context.Background(), "pg") + must(t, err) + if got := h.with("restic backup"); len(got) != 1 || got[0] != "restic backup --json --tag module=pg --tag hand /" { + t.Errorf("backed up by hand with %q, want it tagged hand", got) + } + _, err = b.BackUp(context.Background(), "") + must(t, err) + if got := h.with("restic backup"); len(got) != 2 || got[1] != "restic backup --json --tag module=pg /" { + t.Errorf("the night backed up with %q, want it tagged with its module alone", got) + } + want := "restic forget --prune --group-by host,tags --keep-daily 14 --keep-weekly 8 --keep-monthly 6 --keep-tag hand" + for _, f := range h.with("restic forget") { + if f != want { + t.Errorf("thinned with %q, want %q", f, want) + } + // No rule that would keep every night as well: the hand tag is the only thing kept beyond the counts. + if strings.Contains(f, "--keep-within") || strings.Contains(f, "--keep-last") { + t.Errorf("thinned with %q, a rule that keeps nights too", f) + } + } + if len(h.with("restic forget")) != 2 { + t.Errorf("thinned %d times, want after each of the two backups", len(h.with("restic forget"))) + } +} + +func TestAHandTakenRestorePointIsLetGoOnlyPastItsBoundAndAfterALaterNight(t *testing.T) { + now := time.Date(2026, 10, 20, 3, 0, 0, 0, time.UTC) + at := func(days float64) time.Time { return now.Add(-time.Duration(days * 24 * float64(time.Hour))) } + snap := func(id string, when time.Time, tags ...string) Snapshot { + return Snapshot{ID: id + "0000000000", ShortID: id, Time: when, Tags: tags} + } + snaps := []Snapshot{ + snap("oldhand1", at(10), "module=pg", HandTag), // past the bound, a night after it: let go + snap("night001", at(9), "module=pg"), // the night after it + snap("freshhnd", at(2), "module=pg", HandTag), // within the bound: kept + snap("lonehand", at(12), "module=mail", HandTag), // past the bound, no night of mail after it: kept + snap("mailnite", at(13), "module=mail"), // a night of mail, but before it + snap("pghand02", at(8.5), "module=pg", HandTag), // past the bound, night001 is before it, but a night of + snap("pgnight2", at(1), "module=pg"), // pg after it exists: let go + snap("retired1", at(30), "module=pg", "retired=data"), + snap("otherhnd", at(11), "module=web", HandTag), // a night of another module after it is no night of its own + snap("ret-hand", at(11), "module=web", "retired=x"), // a retired restore point is no night either + snap("webnight", at(10.5), "module=web", "retired=data"), // nor this + } + got := handLetGo(snaps, now, 7) + var ids []string + for _, s := range got { + ids = append(ids, s.ShortID) + } + slices.Sort(ids) + if want := []string{"oldhand1", "pghand02"}; !slices.Equal(ids, want) { + t.Errorf("let go %v, want %v", ids, want) + } +} + +// The thinning forgets a hand-taken restore point past its bound by its id, and records which. +func TestTheThinningForgetsAnExpiredHandTakenRestorePointByItsId(t *testing.T) { + now := time.Date(2026, 10, 20, 3, 0, 0, 0, time.UTC) + h := &handRestic{snaps: []Snapshot{ + {ID: "aaaaaaaa11", ShortID: "aaaaaaaa", Time: now.AddDate(0, 0, -9), Tags: []string{"module=pg", HandTag}}, + {ID: "bbbbbbbb11", ShortID: "bbbbbbbb", Time: now, Tags: []string{"module=pg"}}, + }} + b := &Backups{Where: placed(t, "# pg\npath /\n"), Run: h.run, Now: func() time.Time { return now }, Say: quiet} + _, err := b.BackUp(context.Background(), "") + must(t, err) + forgets := h.with("restic forget") + if len(forgets) != 2 || forgets[0] != "restic forget aaaaaaaa11" || !strings.HasPrefix(forgets[1], "restic forget --prune") { + t.Fatalf("thinned with %q, want the expired one forgotten by id, then the rotation", forgets) + } + th := b.LastRun().Thinned + if th == nil || !th.OK || !slices.Equal(th.LetGo, []string{"aaaaaaaa"}) { + t.Errorf("recorded %+v, want aaaaaaaa let go", th) + } +} + +// When the restore points cannot be listed, nothing is thinned: a hand-taken one is never lost to a list +// that did not read. +func TestARestorePointListThatFailsThinsNothing(t *testing.T) { + var forgot []string + run := func(_ context.Context, name string, args ...string) (string, error) { + if name != "restic" { + return "", nil + } + switch args[5] { + case "backup": + return "{\"message_type\":\"summary\",\"snapshot_id\":\"abcdef0123456789\"}\n", nil + case "snapshots": + return "", errors.New("Fatal: repository is already locked") + case "forget": + forgot = append(forgot, strings.Join(args[5:], " ")) + } + return "", nil + } + b := &Backups{Where: placed(t, "# pg\npath /\n"), Run: run, Now: time.Now, Say: quiet} + _, err := b.BackUp(context.Background(), "") + must(t, err) + if len(forgot) != 0 { + t.Errorf("thinned with %q", forgot) + } + if th := b.LastRun().Thinned; th == nil || th.OK || !strings.Contains(th.Error, "already locked") { + t.Errorf("recorded %+v, want the failure", th) + } +} + +// backed-up says each hand-taken restore point and until when it is kept. +func TestBackedUpSaysWhichRestorePointsAreKeptAsHandTakenAndUntilWhen(t *testing.T) { + now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC) + early, late := now.Add(-10*time.Hour), now.Add(-time.Hour) + h := &handRestic{snaps: []Snapshot{ + {ID: "aaaaaaaa11", ShortID: "aaaaaaaa", Time: early, Tags: []string{"module=pg", HandTag}}, + {ID: "bbbbbbbb11", ShortID: "bbbbbbbb", Time: now.Add(-5 * time.Hour), Tags: []string{"module=pg"}}, + {ID: "cccccccc11", ShortID: "cccccccc", Time: late, Tags: []string{"module=pg", HandTag}}, + }} + b := &Backups{Where: placed(t, "# pg\npath /\n"), Run: h.run, Now: func() time.Time { return now }, Say: quiet} + listed, err := b.BackedUp(context.Background(), "pg") + must(t, err) + if len(listed) != 1 || len(listed[0].HandTaken) != 2 { + t.Fatalf("listed %+v, want pg with two hand-taken restore points", listed) + } + first, second := listed[0].HandTaken[0], listed[0].HandTaken[1] + if first.Snapshot != "aaaaaaaa" || !first.KeptAtLeastUntil.Equal(early.AddDate(0, 0, 7)) || first.WaitsForANight { + t.Errorf("the first: %+v, want kept until 7 days after it, a night of pg after it already there", first) + } + if second.Snapshot != "cccccccc" || !second.KeptAtLeastUntil.Equal(late.AddDate(0, 0, 7)) || !second.WaitsForANight { + t.Errorf("the second: %+v, want kept at least 7 days and until a later night", second) + } + if !strings.Contains(second.Kept, "a later good night of pg") || !strings.Contains(first.Kept, early.AddDate(0, 0, 7).Format(time.RFC3339)) { + t.Errorf("said %q and %q", first.Kept, second.Kept) + } +} + +// The real thing, where restic is installed: the issue as it was seen — a backup taken by hand, then a night +// the same day — and the hand-taken restore point is still there; a week and a day later, with a later night +// of its module, the thinning lets it go. +func TestWithTheRealResticAHandTakenBackupOutlivesTheNightTheSameDay(t *testing.T) { + if _, err := exec.LookPath("restic"); err != nil { + t.Skip("restic is not installed") + } + root := t.TempDir() + data := filepath.Join(root, "data") + must(t, os.Mkdir(data, 0o700)) + must(t, os.WriteFile(filepath.Join(data, "library"), []byte("before the migration\n"), 0o600)) + where := placed(t, "# plex\npath "+data+"\n") + // A machine whose counts are full, as ace's plex was: restic keeps the oldest restore point of a period not + // yet filled, which would hide the issue on a fresh repository. + conf, err := os.ReadFile(filepath.Join(where.State, RetentionFile)) + must(t, err) + must(t, os.WriteFile(filepath.Join(where.State, RetentionFile), + []byte(strings.Replace(string(conf), "daily=14\nweekly=8\nmonthly=6", "daily=1\nweekly=1\nmonthly=1", 1)), 0o600)) + // As in the other real test: the directories above the test's own are said to be root's, so the restore + // beside is what is tested. + run := func(ctx context.Context, name string, args ...string) (string, error) { + if name == "stat" && len(args) > 0 && strings.HasPrefix(root, strings.TrimSuffix(args[len(args)-1], "/")+"/") { + return "0 755 directory\n", nil + } + out, err := exec.CommandContext(ctx, name, args...).Output() + if ee, ok := err.(*exec.ExitError); ok { + return string(out), errors.New(string(ee.Stderr)) + } + return string(out), err + } + clock := time.Now() + b := &Backups{Where: where, Run: run, Now: func() time.Time { return clock }, Say: quiet} + ctx := context.Background() + + hand, err := b.BackUpNow(ctx, "plex") + must(t, err) + must(t, os.WriteFile(filepath.Join(data, "library"), []byte("the damage\n"), 0o600)) + night, err := b.BackUp(ctx, "") + must(t, err) + if !hand["plex"].OK || !night["plex"].OK { + t.Fatalf("hand %+v, night %+v", hand, night) + } + listed, err := b.BackedUp(ctx, "plex") + must(t, err) + if listed[0].RestorePoints != 2 { + t.Fatalf("%d restore points after the night, want the hand-taken one kept beside the night's", listed[0].RestorePoints) + } + if len(listed[0].HandTaken) != 1 || listed[0].HandTaken[0].Snapshot != hand["plex"].Snapshot { + t.Fatalf("hand-taken %+v, want %s", listed[0].HandTaken, hand["plex"].Snapshot) + } + r, err := b.Restore(ctx, "plex", hand["plex"].Snapshot, "") + must(t, err) + if raw, _ := os.ReadFile(filepath.Join(r.Restored[0], "library")); string(raw) != "before the migration\n" { + t.Errorf("the hand-taken restore point holds %q", raw) + } + + // Eight days on, a night of plex later than it exists: the thinning lets it go. + clock = clock.AddDate(0, 0, 8) + _, err = b.BackUp(ctx, "") + must(t, err) + listed, err = b.BackedUp(ctx, "plex") + must(t, err) + if len(listed[0].HandTaken) != 0 { + t.Errorf("hand-taken %+v eight days on, want it let go", listed[0].HandTaken) + } + if th := b.LastRun().Thinned; th == nil || !slices.Equal(th.LetGo, []string{hand["plex"].Snapshot}) { + t.Errorf("recorded %+v, want the hand-taken restore point let go", th) + } +} diff --git a/modules/restic/cmd/restic-backups/health.go b/modules/restic/cmd/restic-backups/health.go index 1fd71ec0..86561f4e 100644 --- a/modules/restic/cmd/restic-backups/health.go +++ b/modules/restic/cmd/restic-backups/health.go @@ -113,8 +113,8 @@ func Judge(s Seen) Health { r := s.Retention switch { case r.Refused == "" && r.Retention != nil: - said = append(said, fmt.Sprintf("The next thinning keeps %d daily, %d weekly and %d monthly restore points of each module.", - r.Daily, r.Weekly, r.Monthly)) + said = append(said, fmt.Sprintf("The next thinning keeps %d daily, %d weekly and %d monthly restore points of each module, "+ + "and each taken by hand until a later good night and %d days at least.", r.Daily, r.Weekly, r.Monthly, r.Hand)) case r.Setting != "": value := "not set" if r.Value != "" { @@ -138,7 +138,7 @@ func Judge(s Seen) Health { case strings.Contains(r.Refused, RetentionFile+": "): // Read, and not in the form the mesh writes: something other than the mesh wrote it. found = append(found, fmt.Sprintf("The retention is refused: %s; nothing is thinned and every restore point is "+ - "kept, so the repository grows. The mesh writes only daily=, weekly= and monthly=, so something else wrote "+ + "kept, so the repository grows. The mesh writes only daily=, weekly=, monthly= and hand=, so something else wrote "+ "this file; push this machine (mesh-controller.push) so the mesh writes it again.", r.Refused)) default: found = append(found, fmt.Sprintf("The retention is refused: %s; nothing is thinned and every restore point is "+ @@ -173,8 +173,8 @@ func healthySaid(s Seen, said []string) string { case t == nil: parts = append(parts, "No thinning is recorded yet.") case t.OK && t.Kept != nil: - parts = append(parts, fmt.Sprintf("The last thinning, at %s, kept %d daily, %d weekly and %d monthly restore points.", - clock(t.At), t.Kept.Daily, t.Kept.Weekly, t.Kept.Monthly)) + parts = append(parts, fmt.Sprintf("The last thinning, at %s, kept %d daily, %d weekly and %d monthly restore points%s.", + clock(t.At), t.Kept.Daily, t.Kept.Weekly, t.Kept.Monthly, letGoSaid(t.LetGo))) case t.Refused != "": parts = append(parts, fmt.Sprintf("The last thinning, at %s, thinned nothing: its retention was refused then.", clock(t.At))) } @@ -187,3 +187,11 @@ func (b *Backups) Health(ctx context.Context) Health { return Judge(Seen{Declared: declared, DeclaredErr: err, Nights: b.Nights(), Good: b.Good(), Run: b.LastRun(), Retention: b.inForce(ctx), Awake: b.LastAwake(), Now: b.Now()}) } + +// letGoSaid is the restore points taken by hand a thinning let go, as the health says them (novox/hq issue 453). +func letGoSaid(ids []string) string { + if len(ids) == 0 { + return "" + } + return ", and let go " + strings.Join(ids, ", ") + ", taken by hand, past their days and a later night" +} diff --git a/modules/restic/cmd/restic-backups/health_test.go b/modules/restic/cmd/restic-backups/health_test.go index 452e4e4e..ee43067a 100644 --- a/modules/restic/cmd/restic-backups/health_test.go +++ b/modules/restic/cmd/restic-backups/health_test.go @@ -363,7 +363,7 @@ func TestAHealthyAnswerSaysOnlyWhatWasSeen(t *testing.T) { if h := Judge(seen); !strings.Contains(h.Why, "thinned nothing") || strings.Contains(h.Why, "kept 14") { t.Fatalf("after a refused thinning: %q", h.Why) } - seen.Run.Thinned = &Thinning{At: good, OK: true, Kept: &Retention{7, 4, 24}} + seen.Run.Thinned = &Thinning{At: good, OK: true, Kept: &Retention{7, 4, 24, 10}} if h := Judge(seen); !strings.Contains(h.Why, "kept 7 daily, 4 weekly and 24 monthly") { t.Fatalf("after a thinning: %q", h.Why) } diff --git a/modules/restic/cmd/restic-backups/main.go b/modules/restic/cmd/restic-backups/main.go index 2e9d4610..37493adf 100644 --- a/modules/restic/cmd/restic-backups/main.go +++ b/modules/restic/cmd/restic-backups/main.go @@ -148,13 +148,13 @@ func tools(b *Backups) []stdio.Tool { verb("backed-up", "What this machine backs up: each module, what it declared, its last good night, how many restore points are kept.", map[string]any{"module": str("one module (optional)")}, func(a map[string]any) (any, error) { return b.BackedUp(context.Background(), arg(a, "module")) }), - verb("now", "Take a backup now, of one module or of every module on this machine — before a migration, a retirement or anything else that could go wrong. Answers when it has started; `backed-up` says how it went.", + verb("now", "Take a backup now, of one module or of every module on this machine — before a migration, a retirement or anything else that could go wrong. Kept as taken by hand until a later good night of the module exists, and for keep-hand-taken-days in any case. Answers when it has started; `backed-up` says how it went.", map[string]any{"module": str("one module (optional)")}, func(a map[string]any) (any, error) { only := arg(a, "module") // A night of a large store outlasts any call; it is started, and its outcome recorded. go func() { - if _, err := b.BackUp(context.Background(), only); err != nil { + if _, err := b.BackUpNow(context.Background(), only); err != nil { say("a backup asked for now failed: %v", err) } }() diff --git a/modules/restic/cmd/restic-backups/retention.go b/modules/restic/cmd/restic-backups/retention.go index 26d23bf7..9ae90aed 100644 --- a/modules/restic/cmd/restic-backups/retention.go +++ b/modules/restic/cmd/restic-backups/retention.go @@ -1,7 +1,7 @@ package main // The retention as a preference (novox/hq issue 416, ADR 0262). The mesh writes the settings -// keep-daily, keep-weekly and keep-monthly into RetentionFile in this module's state, each defaulting +// keep-daily, keep-weekly and keep-monthly, and keep-hand-taken-days (novox/hq issue 453), into RetentionFile in this module's state, each defaulting // to today's rotation, so a machine that sets none thins exactly as before. The controller does not // type a setting yet (ADR 0164 is proposed), so this program is where a bad value is caught: it is // refused here and never handed to restic (issue 346: a raw setting is an injection point). @@ -18,16 +18,19 @@ import ( "strings" ) -// Retention is how many restore points each period keeps. +// Retention is how many restore points each period keeps, and how many days a restore point taken by hand is +// kept at least (hand.go). type Retention struct { Daily int `json:"daily"` Weekly int `json:"weekly"` Monthly int `json:"monthly"` + Hand int `json:"handDays"` } // TodaysRetention is the rotation before it was a preference (novox/hq ADR 0214), and still the -// manifest's default — never a fallback for a file that is not there (novox/hq issue 454). -var TodaysRetention = Retention{14, 8, 6} +// manifest's default — never a fallback for a file that is not there (novox/hq issue 454). A restore point +// taken by hand is kept a week at least: long enough to notice what a change broke (novox/hq issue 453). +var TodaysRetention = Retention{14, 8, 6, 7} // RetentionFile is the file the mesh writes the retention into, in this module's state. const RetentionFile = "retention.conf" @@ -36,10 +39,11 @@ const RetentionFile = "retention.conf" // past four digits is a typo, not a policy. var count = regexp.MustCompile(`^[1-9][0-9]{0,3}$`) -// ParseRetention reads daily=, weekly= and monthly=, each once and each a count; anything else is +// ParseRetention reads daily=, weekly=, monthly= and hand=, each once and each a count; anything else is // refused naming the setting it came from. func ParseRetention(text string) (Retention, error) { - settingOf := map[string]string{"daily": "keep-daily", "weekly": "keep-weekly", "monthly": "keep-monthly"} + settingOf := map[string]string{"daily": "keep-daily", "weekly": "keep-weekly", "monthly": "keep-monthly", + "hand": "keep-hand-taken-days"} seen := map[string]int{} for _, l := range strings.Split(text, "\n") { l = strings.TrimSpace(l) @@ -54,7 +58,7 @@ func ParseRetention(text string) (Retention, error) { setting, known := settingOf[k] switch { case !known: - return Retention{}, fmt.Errorf("%q is no retention this module reads (daily, weekly, monthly)", k) + return Retention{}, fmt.Errorf("%q is no retention this module reads (daily, weekly, monthly, hand)", k) case seen[k] != 0: return Retention{}, fmt.Errorf("%s is given twice", k) case !count.MatchString(v): @@ -62,12 +66,12 @@ func ParseRetention(text string) (Retention, error) { } seen[k], _ = strconv.Atoi(v) } - for _, k := range []string{"daily", "weekly", "monthly"} { + for _, k := range []string{"daily", "weekly", "monthly", "hand"} { if seen[k] == 0 { return Retention{}, &SettingRefused{Setting: settingOf[k]} } } - return Retention{seen["daily"], seen["weekly"], seen["monthly"]}, nil + return Retention{seen["daily"], seen["weekly"], seen["monthly"], seen["hand"]}, nil } // SettingRefused is one retention setting the night refuses: the operator's to give again. diff --git a/modules/restic/cmd/restic-backups/retention_test.go b/modules/restic/cmd/restic-backups/retention_test.go index 9b489b80..02f1f237 100644 --- a/modules/restic/cmd/restic-backups/retention_test.go +++ b/modules/restic/cmd/restic-backups/retention_test.go @@ -49,9 +49,9 @@ func thinning(t *testing.T, retention *string) (string, []string) { } func TestANightThinsByTheRetentionTheMeshWrote(t *testing.T) { - conf := "# Written by the mesh.\ndaily=7\nweekly=4\nmonthly=24\n" + conf := "# Written by the mesh.\ndaily=7\nweekly=4\nmonthly=24\nhand=10\n" got, _ := thinning(t, &conf) - if want := "restic forget --prune --group-by host,tags --keep-daily 7 --keep-weekly 4 --keep-monthly 24"; got != want { + if want := "restic forget --prune --group-by host,tags --keep-daily 7 --keep-weekly 4 --keep-monthly 24 --keep-tag hand"; got != want { t.Errorf("ran %q, want %q", got, want) } } @@ -145,6 +145,8 @@ func TestARetentionThatIsNotACountThinsNothingAndSaysWhich(t *testing.T) { {"daily=14\nweekly=8\nmonthly=6\nlast=1\n", "last"}, {"daily=14\ndaily=3\nweekly=8\nmonthly=6\n", "daily"}, {"daily 14\nweekly=8\nmonthly=6\n", "key=value"}, + {"daily=14\nweekly=8\nmonthly=6\nhand=0\n", "keep-hand-taken-days"}, + {"daily=14\nweekly=8\nmonthly=6\n", "keep-hand-taken-days"}, } { got, said := thinning(t, &c.conf) if got != "" { @@ -171,7 +173,7 @@ func TestTheManifestsDefaultsAreTodaysRetention(t *testing.T) { Resources []map[string]any `json:"resources"` } must(t, json.Unmarshal(raw, &m)) - want := map[string]float64{"keep-daily": 14, "keep-weekly": 8, "keep-monthly": 6} + want := map[string]float64{"keep-daily": 14, "keep-weekly": 8, "keep-monthly": 6, "keep-hand-taken-days": 7} for k, v := range want { s, ok := m.Settings[k] if !ok || s.Kind != "preference" || s.Default != v || s.Why == "" { diff --git a/modules/restic/module.json b/modules/restic/module.json index 544c0b81..604ce0c7 100644 --- a/modules/restic/module.json +++ b/modules/restic/module.json @@ -52,6 +52,11 @@ "kind": "preference", "default": 6, "why": "half a year of monthly restore points behind the weeklies; a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone" + }, + "keep-hand-taken-days": { + "kind": "preference", + "default": 7, + "why": "a restore point taken by hand (node-backup.now, or the mesh's before a module moves) is kept until a later good night of its module exists, and this many days in any case, so the next thinning never takes the one taken against a risky change (novox/hq issue 453); a count from 1 to 9999, refused by the night like keep-daily; lowering it lets go of the restore points taken by hand that are older than the new count, past a later night, at the next thinning, and that cannot be undone" } }, "resources": [ @@ -86,7 +91,7 @@ "path": "${dir:state}/retention.conf", "mode": "0644", "trusted": true, - "content": "# How many restore points the night keeps, written by the mesh from this machine's settings keep-daily,\n# keep-weekly and keep-monthly (novox/hq issue 416). Replaced on every push; change the settings, never this file.\ndaily=${setting:keep-daily}\nweekly=${setting:keep-weekly}\nmonthly=${setting:keep-monthly}\n" + "content": "# How many restore points the night keeps, written by the mesh from this machine's settings keep-daily,\n# keep-weekly, keep-monthly (novox/hq issue 416) and keep-hand-taken-days (novox/hq issue 453). Replaced on every push; change the settings, never this file.\ndaily=${setting:keep-daily}\nweekly=${setting:keep-weekly}\nmonthly=${setting:keep-monthly}\nhand=${setting:keep-hand-taken-days}\n" }, { "id": "tool",