jschoubben
879bc4bca2
mesh/merge-gate pass: builds restic → ace, g14, novox, shanks; no bus step; every machine composes with the change as it did without (4 of 4 compose)
mesh/repo-check pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-check-typescript.sh (typescript)
mesh/delivery superseded: a newer head of the same pull request
257 lines
11 KiB
Go
257 lines
11 KiB
Go
package main
|
|
|
|
// A backup taken by hand — `node-backup.now`, asked by a person or by the mesh before a module moves (novox/hq
|
|
// ADR 0233) — is kept by the thinning as its own restore point: until a later good night of the same module
|
|
// exists, and for the machine's keep-hand-taken-days in any case (novox/hq issue 453: one taken at 02:35 was
|
|
// forgotten by the night at 03:01, as keep-daily keeps only the newest restore point of each day).
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// handRestic is a fake restic that remembers each command and lists the snapshots it is given.
|
|
type handRestic struct {
|
|
calls []string
|
|
snaps []Snapshot
|
|
}
|
|
|
|
func (h *handRestic) run(_ context.Context, name string, args ...string) (string, error) {
|
|
if name != "restic" {
|
|
return "", nil
|
|
}
|
|
line := "restic " + strings.Join(args[5:], " ")
|
|
h.calls = append(h.calls, line)
|
|
switch args[5] {
|
|
case "backup":
|
|
return "{\"message_type\":\"summary\",\"snapshot_id\":\"abcdef0123456789\"}\n", nil
|
|
case "snapshots":
|
|
raw, _ := json.Marshal(h.snaps)
|
|
return string(raw), nil
|
|
}
|
|
return "", nil
|
|
}
|
|
|
|
func (h *handRestic) with(prefix string) []string {
|
|
var out []string
|
|
for _, c := range h.calls {
|
|
if strings.HasPrefix(c, prefix) {
|
|
out = append(out, c)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func TestAHandTakenBackupIsTaggedAndTheThinningKeepsThatTag(t *testing.T) {
|
|
h := &handRestic{}
|
|
b := &Backups{Where: placed(t, "# pg\npath /\n"), Run: h.run,
|
|
Now: func() time.Time { return time.Date(2026, 10, 11, 2, 35, 0, 0, time.UTC) }, Say: quiet}
|
|
_, err := b.BackUpNow(context.Background(), "pg")
|
|
must(t, err)
|
|
if got := h.with("restic backup"); len(got) != 1 || got[0] != "restic backup --json --tag module=pg --tag hand /" {
|
|
t.Errorf("backed up by hand with %q, want it tagged hand", got)
|
|
}
|
|
_, err = b.BackUp(context.Background(), "")
|
|
must(t, err)
|
|
if got := h.with("restic backup"); len(got) != 2 || got[1] != "restic backup --json --tag module=pg /" {
|
|
t.Errorf("the night backed up with %q, want it tagged with its module alone", got)
|
|
}
|
|
want := "restic forget --prune --group-by host,tags --keep-daily 14 --keep-weekly 8 --keep-monthly 6 --keep-tag hand"
|
|
for _, f := range h.with("restic forget") {
|
|
if f != want {
|
|
t.Errorf("thinned with %q, want %q", f, want)
|
|
}
|
|
// No rule that would keep every night as well: the hand tag is the only thing kept beyond the counts.
|
|
if strings.Contains(f, "--keep-within") || strings.Contains(f, "--keep-last") {
|
|
t.Errorf("thinned with %q, a rule that keeps nights too", f)
|
|
}
|
|
}
|
|
if len(h.with("restic forget")) != 2 {
|
|
t.Errorf("thinned %d times, want after each of the two backups", len(h.with("restic forget")))
|
|
}
|
|
}
|
|
|
|
func TestAHandTakenRestorePointIsLetGoOnlyPastItsBoundAndAfterALaterNight(t *testing.T) {
|
|
now := time.Date(2026, 10, 20, 3, 0, 0, 0, time.UTC)
|
|
at := func(days float64) time.Time { return now.Add(-time.Duration(days * 24 * float64(time.Hour))) }
|
|
snap := func(id string, when time.Time, tags ...string) Snapshot {
|
|
return Snapshot{ID: id + "0000000000", ShortID: id, Time: when, Tags: tags}
|
|
}
|
|
snaps := []Snapshot{
|
|
snap("oldhand1", at(10), "module=pg", HandTag), // past the bound, a night after it: let go
|
|
snap("night001", at(9), "module=pg"), // the night after it
|
|
snap("freshhnd", at(2), "module=pg", HandTag), // within the bound: kept
|
|
snap("lonehand", at(12), "module=mail", HandTag), // past the bound, no night of mail after it: kept
|
|
snap("mailnite", at(13), "module=mail"), // a night of mail, but before it
|
|
snap("pghand02", at(8.5), "module=pg", HandTag), // past the bound, night001 is before it, but a night of
|
|
snap("pgnight2", at(1), "module=pg"), // pg after it exists: let go
|
|
snap("retired1", at(30), "module=pg", "retired=data"),
|
|
snap("otherhnd", at(11), "module=web", HandTag), // a night of another module after it is no night of its own
|
|
snap("ret-hand", at(11), "module=web", "retired=x"), // a retired restore point is no night either
|
|
snap("webnight", at(10.5), "module=web", "retired=data"), // nor this
|
|
}
|
|
got := handLetGo(snaps, now, 7)
|
|
var ids []string
|
|
for _, s := range got {
|
|
ids = append(ids, s.ShortID)
|
|
}
|
|
slices.Sort(ids)
|
|
if want := []string{"oldhand1", "pghand02"}; !slices.Equal(ids, want) {
|
|
t.Errorf("let go %v, want %v", ids, want)
|
|
}
|
|
}
|
|
|
|
// The thinning forgets a hand-taken restore point past its bound by its id, and records which.
|
|
func TestTheThinningForgetsAnExpiredHandTakenRestorePointByItsId(t *testing.T) {
|
|
now := time.Date(2026, 10, 20, 3, 0, 0, 0, time.UTC)
|
|
h := &handRestic{snaps: []Snapshot{
|
|
{ID: "aaaaaaaa11", ShortID: "aaaaaaaa", Time: now.AddDate(0, 0, -9), Tags: []string{"module=pg", HandTag}},
|
|
{ID: "bbbbbbbb11", ShortID: "bbbbbbbb", Time: now, Tags: []string{"module=pg"}},
|
|
}}
|
|
b := &Backups{Where: placed(t, "# pg\npath /\n"), Run: h.run, Now: func() time.Time { return now }, Say: quiet}
|
|
_, err := b.BackUp(context.Background(), "")
|
|
must(t, err)
|
|
forgets := h.with("restic forget")
|
|
if len(forgets) != 2 || forgets[0] != "restic forget aaaaaaaa11" || !strings.HasPrefix(forgets[1], "restic forget --prune") {
|
|
t.Fatalf("thinned with %q, want the expired one forgotten by id, then the rotation", forgets)
|
|
}
|
|
th := b.LastRun().Thinned
|
|
if th == nil || !th.OK || !slices.Equal(th.LetGo, []string{"aaaaaaaa"}) {
|
|
t.Errorf("recorded %+v, want aaaaaaaa let go", th)
|
|
}
|
|
}
|
|
|
|
// When the restore points cannot be listed, nothing is thinned: a hand-taken one is never lost to a list
|
|
// that did not read.
|
|
func TestARestorePointListThatFailsThinsNothing(t *testing.T) {
|
|
var forgot []string
|
|
run := func(_ context.Context, name string, args ...string) (string, error) {
|
|
if name != "restic" {
|
|
return "", nil
|
|
}
|
|
switch args[5] {
|
|
case "backup":
|
|
return "{\"message_type\":\"summary\",\"snapshot_id\":\"abcdef0123456789\"}\n", nil
|
|
case "snapshots":
|
|
return "", errors.New("Fatal: repository is already locked")
|
|
case "forget":
|
|
forgot = append(forgot, strings.Join(args[5:], " "))
|
|
}
|
|
return "", nil
|
|
}
|
|
b := &Backups{Where: placed(t, "# pg\npath /\n"), Run: run, Now: time.Now, Say: quiet}
|
|
_, err := b.BackUp(context.Background(), "")
|
|
must(t, err)
|
|
if len(forgot) != 0 {
|
|
t.Errorf("thinned with %q", forgot)
|
|
}
|
|
if th := b.LastRun().Thinned; th == nil || th.OK || !strings.Contains(th.Error, "already locked") {
|
|
t.Errorf("recorded %+v, want the failure", th)
|
|
}
|
|
}
|
|
|
|
// backed-up says each hand-taken restore point and until when it is kept.
|
|
func TestBackedUpSaysWhichRestorePointsAreKeptAsHandTakenAndUntilWhen(t *testing.T) {
|
|
now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC)
|
|
early, late := now.Add(-10*time.Hour), now.Add(-time.Hour)
|
|
h := &handRestic{snaps: []Snapshot{
|
|
{ID: "aaaaaaaa11", ShortID: "aaaaaaaa", Time: early, Tags: []string{"module=pg", HandTag}},
|
|
{ID: "bbbbbbbb11", ShortID: "bbbbbbbb", Time: now.Add(-5 * time.Hour), Tags: []string{"module=pg"}},
|
|
{ID: "cccccccc11", ShortID: "cccccccc", Time: late, Tags: []string{"module=pg", HandTag}},
|
|
}}
|
|
b := &Backups{Where: placed(t, "# pg\npath /\n"), Run: h.run, Now: func() time.Time { return now }, Say: quiet}
|
|
listed, err := b.BackedUp(context.Background(), "pg")
|
|
must(t, err)
|
|
if len(listed) != 1 || len(listed[0].HandTaken) != 2 {
|
|
t.Fatalf("listed %+v, want pg with two hand-taken restore points", listed)
|
|
}
|
|
first, second := listed[0].HandTaken[0], listed[0].HandTaken[1]
|
|
if first.Snapshot != "aaaaaaaa" || !first.KeptAtLeastUntil.Equal(early.AddDate(0, 0, 7)) || first.WaitsForANight {
|
|
t.Errorf("the first: %+v, want kept until 7 days after it, a night of pg after it already there", first)
|
|
}
|
|
if second.Snapshot != "cccccccc" || !second.KeptAtLeastUntil.Equal(late.AddDate(0, 0, 7)) || !second.WaitsForANight {
|
|
t.Errorf("the second: %+v, want kept at least 7 days and until a later night", second)
|
|
}
|
|
if !strings.Contains(second.Kept, "a later good night of pg") || !strings.Contains(first.Kept, early.AddDate(0, 0, 7).Format(time.RFC3339)) {
|
|
t.Errorf("said %q and %q", first.Kept, second.Kept)
|
|
}
|
|
}
|
|
|
|
// The real thing, where restic is installed: the issue as it was seen — a backup taken by hand, then a night
|
|
// the same day — and the hand-taken restore point is still there; a week and a day later, with a later night
|
|
// of its module, the thinning lets it go.
|
|
func TestWithTheRealResticAHandTakenBackupOutlivesTheNightTheSameDay(t *testing.T) {
|
|
if _, err := exec.LookPath("restic"); err != nil {
|
|
t.Skip("restic is not installed")
|
|
}
|
|
root := t.TempDir()
|
|
data := filepath.Join(root, "data")
|
|
must(t, os.Mkdir(data, 0o700))
|
|
must(t, os.WriteFile(filepath.Join(data, "library"), []byte("before the migration\n"), 0o600))
|
|
where := placed(t, "# plex\npath "+data+"\n")
|
|
// A machine whose counts are full, as ace's plex was: restic keeps the oldest restore point of a period not
|
|
// yet filled, which would hide the issue on a fresh repository.
|
|
conf, err := os.ReadFile(filepath.Join(where.State, RetentionFile))
|
|
must(t, err)
|
|
must(t, os.WriteFile(filepath.Join(where.State, RetentionFile),
|
|
[]byte(strings.Replace(string(conf), "daily=14\nweekly=8\nmonthly=6", "daily=1\nweekly=1\nmonthly=1", 1)), 0o600))
|
|
// As in the other real test: the directories above the test's own are said to be root's, so the restore
|
|
// beside is what is tested.
|
|
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
|
if name == "stat" && len(args) > 0 && strings.HasPrefix(root, strings.TrimSuffix(args[len(args)-1], "/")+"/") {
|
|
return "0 755 directory\n", nil
|
|
}
|
|
out, err := exec.CommandContext(ctx, name, args...).Output()
|
|
if ee, ok := err.(*exec.ExitError); ok {
|
|
return string(out), errors.New(string(ee.Stderr))
|
|
}
|
|
return string(out), err
|
|
}
|
|
clock := time.Now()
|
|
b := &Backups{Where: where, Run: run, Now: func() time.Time { return clock }, Say: quiet}
|
|
ctx := context.Background()
|
|
|
|
hand, err := b.BackUpNow(ctx, "plex")
|
|
must(t, err)
|
|
must(t, os.WriteFile(filepath.Join(data, "library"), []byte("the damage\n"), 0o600))
|
|
night, err := b.BackUp(ctx, "")
|
|
must(t, err)
|
|
if !hand["plex"].OK || !night["plex"].OK {
|
|
t.Fatalf("hand %+v, night %+v", hand, night)
|
|
}
|
|
listed, err := b.BackedUp(ctx, "plex")
|
|
must(t, err)
|
|
if listed[0].RestorePoints != 2 {
|
|
t.Fatalf("%d restore points after the night, want the hand-taken one kept beside the night's", listed[0].RestorePoints)
|
|
}
|
|
if len(listed[0].HandTaken) != 1 || listed[0].HandTaken[0].Snapshot != hand["plex"].Snapshot {
|
|
t.Fatalf("hand-taken %+v, want %s", listed[0].HandTaken, hand["plex"].Snapshot)
|
|
}
|
|
r, err := b.Restore(ctx, "plex", hand["plex"].Snapshot, "")
|
|
must(t, err)
|
|
if raw, _ := os.ReadFile(filepath.Join(r.Restored[0], "library")); string(raw) != "before the migration\n" {
|
|
t.Errorf("the hand-taken restore point holds %q", raw)
|
|
}
|
|
|
|
// Eight days on, a night of plex later than it exists: the thinning lets it go.
|
|
clock = clock.AddDate(0, 0, 8)
|
|
_, err = b.BackUp(ctx, "")
|
|
must(t, err)
|
|
listed, err = b.BackedUp(ctx, "plex")
|
|
must(t, err)
|
|
if len(listed[0].HandTaken) != 0 {
|
|
t.Errorf("hand-taken %+v eight days on, want it let go", listed[0].HandTaken)
|
|
}
|
|
if th := b.LastRun().Thinned; th == nil || !slices.Equal(th.LetGo, []string{hand["plex"].Snapshot}) {
|
|
t.Errorf("recorded %+v, want the hand-taken restore point let go", th)
|
|
}
|
|
}
|