package main // The data the modules declare, measured, and a retired item deleted only after a last restore point // (novox/hq ADR 0233) — over a fake restic and a fake machine. import ( "context" "errors" "fmt" "os" "path/filepath" "strings" "sync" "testing" "time" ) const composedData = "# The data the modules on this machine declare, composed by the mesh. Do not edit.\n" + "# postgres\nitem store irreplaceable /var/lib/mesh-store /var/lib/mesh-store/dumps\nitem dumps rebuildable /var/lib/mesh-store/dumps -\n" + "# searxng\nitem valkey cache /var/lib/searxng/valkey-data -\n" func withData(t *testing.T, declared, data string) Where { t.Helper() w := placed(t, declared) w.Data = filepath.Join(w.State, "data.conf") must(t, os.WriteFile(w.Data, []byte(data), 0o600)) return w } func TestTheComposedDataFileIsReadIntoEachModulesItems(t *testing.T) { got, err := parseItems(composedData) must(t, err) if len(got["postgres"]) != 2 || got["postgres"][0].CoveredBy != "/var/lib/mesh-store/dumps" || got["postgres"][1].CoveredBy != "" || got["searxng"][0].Class != "cache" { t.Fatalf("%+v", got) } for _, bad := range []string{"# pg\nitem a irreplaceable relative -\n", "# pg\nitem a irreplaceable /x\n", "# pg\nitem a b /x; rm -rf / -\n"} { if _, err := parseItems(bad); err == nil || !strings.Contains(err.Error(), "pg declares a data line") { t.Errorf("%q: %v", bad, err) } } // An older controller composes no data file: nothing is measured, nothing fails. b := &Backups{Where: placed(t, composed), Now: time.Now, Say: quiet} if items, err := b.Items(); err != nil || len(items) != 0 { t.Fatalf("%v, %v", items, err) } } // Every item but a cache is measured — its files' size and its newest change, in one walk as root — and // `backed-up` says each with the newest good night of the module that keeps the path covering it. func TestEveryItemButACacheIsMeasuredAndSaidWithItsLastGoodBackup(t *testing.T) { where := withData(t, composed, composedData) var mu sync.Mutex var walked []string run := func(_ context.Context, name string, args ...string) (string, error) { mu.Lock() defer mu.Unlock() switch { case name == "test": return "", nil case name == "bash" && strings.Contains(args[1], "find '"): walked = append(walked, args[1]) if strings.Contains(args[1], "'/var/lib/mesh-store'") { return "1073741824 1759744800 4000\n0\n", nil } return "5000 1759700000 3\n0\n", nil case name == "restic": return "[]", nil } return "", nil } night := time.Date(2026, 10, 6, 3, 0, 0, 0, time.UTC) b := &Backups{Where: where, Run: run, Now: func() time.Time { return night.Add(time.Hour) }, Say: quiet} must(t, b.MeasureAll(context.Background(), true)) if len(walked) != 2 { t.Fatalf("walked %d paths, want the two that are not a cache: %v", len(walked), walked) } b.recordGood("postgres", night) got, err := b.BackedUp(context.Background(), "") must(t, err) var store, dumps *ItemReport for _, m := range got { for i := range m.Data { switch m.Data[i].ID { case "store": store = &m.Data[i] case "dumps": dumps = &m.Data[i] } } } if store == nil || store.SizeBytes == nil || *store.SizeBytes != 1<<30 || store.LastWrite == nil || store.LastWrite.Unix() != 1759744800 || store.LastBackup == nil || !store.LastBackup.Equal(night) { t.Fatalf("the store is said as %+v", store) } if dumps == nil || dumps.LastBackup != nil { t.Fatalf("an item not backed up is said backed up: %+v", dumps) } } // A retired item is deleted only after a last restore point of it, tagged as retired; a restore point // that fails deletes nothing; and nothing declared now — itself, inside it, or holding it — is deleted. func TestARetiredItemIsDeletedOnlyAfterALastRestorePoint(t *testing.T) { where := withData(t, composed, composedData) var mu sync.Mutex var calls []string failBackup := false run := func(_ context.Context, name string, args ...string) (string, error) { mu.Lock() defer mu.Unlock() line := name + " " + strings.Join(args, " ") if name == "restic" { line = "restic " + strings.Join(args[5:], " ") } calls = append(calls, line) switch { case name == "test": return "", nil case strings.HasPrefix(line, "restic backup"): if failBackup { return "", errors.New("the repository is locked") } return "{\"message_type\":\"summary\",\"snapshot_id\":\"0123456789abcdef\"}\n", nil } return "", nil } b := &Backups{Where: where, Run: run, Now: time.Now, Say: quiet} ctx := context.Background() for _, refused := range []struct{ path, want string }{ {"/var/lib/mesh-store", "declared now"}, {"/var/lib/mesh-store/dumps/old", "declared now"}, {"/var/lib", "declared now"}, {"/var/lib/mailu/data-mail", "backed up now"}, {"/srv", "too near the root"}, {"relative/x", "not a path"}, } { if _, err := b.DeleteRetired(ctx, "m", "i", refused.path, "i", "op", "tidy"); err == nil || !strings.Contains(err.Error(), refused.want) { t.Errorf("%s: %v, want %q", refused.path, err, refused.want) } } if _, err := b.DeleteRetired(ctx, "house", "config", "/var/lib/house/config", "nope", "op", "tidy"); err == nil { t.Error("a deletion without the item's name again was started") } if _, err := b.DeleteRetired(ctx, "house", "config", "/var/lib/house/config", "config", "op", ""); err == nil { t.Error("a deletion without why was started") } wait := func(path string) Deletion { t.Helper() for i := 0; i < 200; i++ { if d, err := b.DeletedOutcome(path); err == nil && d.Done { return d } time.Sleep(5 * time.Millisecond) } t.Fatalf("the deletion of %s never finished", path) return Deletion{} } mu.Lock() failBackup = true mu.Unlock() _, err := b.DeleteRetired(ctx, "house", "config", "/var/lib/house/config", "config", "op", "moved to the anchor") must(t, err) if d := wait("/var/lib/house/config"); d.OK || !strings.Contains(d.Error, "nothing was deleted") { t.Fatalf("a deletion with no restore point: %+v", d) } mu.Lock() for _, c := range calls { if strings.HasPrefix(c, "rm ") { t.Fatal("it deleted without a last restore point") } } mu.Unlock() mu.Lock() failBackup, calls = false, nil mu.Unlock() _, err = b.DeleteRetired(ctx, "house", "config", "/var/lib/house/config", "config", "op", "moved to the anchor") must(t, err) d := wait("/var/lib/house/config") if !d.OK || d.Snapshot != "01234567" { t.Fatalf("%+v", d) } var backup, rm = -1, -1 mu.Lock() defer mu.Unlock() for i, c := range calls { switch { case c == "restic backup --json --tag module=house --tag retired=config /var/lib/house/config": backup = i case c == "rm -rf --one-file-system -- /var/lib/house/config": rm = i } } if backup < 0 || rm < 0 || rm < backup { t.Fatalf("ran %v", calls) } // Asked again, it answers how it went rather than starting over. again, err := b.DeleteRetired(ctx, "house", "config", "/var/lib/house/config", "config", "op", "moved") if err != nil || !again.Done || !again.OK { t.Fatalf("%+v, %v", again, err) } } // The redundant storage under an item is read: a ZFS pool healthy, then degraded; an md array with a // member missing; and plain storage, which is no redundancy at all. func TestTheRedundantStorageUnderAnItemIsRead(t *testing.T) { health, statusX := "ONLINE\n", "pool 'storage' is healthy\n" fs := "storage/media zfs\n" run := func(_ context.Context, name string, args ...string) (string, error) { switch name { case "findmnt": return fs, nil case "zpool": if args[0] == "list" { return health, nil } return statusX, nil case "cat": return "Personalities : [raid1]\nmd127 : active raid1 sdb1[1] sda1[0]\n 976630464 blocks super 1.2 [2/1] [U_]\n\nunused devices: \n", nil } return "", nil } b := &Backups{Run: run, Now: time.Now, Say: quiet} r := b.redundancyOf(context.Background(), "/storage/media/movies") if r == nil || r.Kind != "zfs" || r.Where != "storage" || r.Healthy == nil || !*r.Healthy { t.Fatalf("a healthy pool: %+v", r) } health, statusX = "DEGRADED\n", " pool: storage\n state: DEGRADED\nstatus: One or more devices has been removed\n" if r := b.redundancyOf(context.Background(), "/storage/media/movies"); r.Healthy == nil || *r.Healthy || !strings.Contains(r.Said, "DEGRADED") { t.Fatalf("a degraded pool: %+v", r) } fs = "/dev/md127 ext4\n" if r := b.redundancyOf(context.Background(), "/srv/x"); r == nil || r.Kind != "md" || r.Healthy == nil || *r.Healthy { t.Fatalf("an array missing a member: %+v", r) } fs = "/dev/nvme0n1p3 ext4\n" if r := b.redundancyOf(context.Background(), "/var/lib/x"); r != nil { t.Fatalf("plain storage read as redundant: %+v", r) } } // Large items are never walked: a dataset is measured from the filesystem's counters and its top-level // entries, a shallow item from its top-level entries only; a walk that meets more than its bound stops // and says so; and a walk runs at most once a day — the hourly round keeps the last one. func TestNothingLargeIsWalkedAndAWalkIsBoundedAndDaily(t *testing.T) { data := "# media\nitem movies irreplaceable /storage/media/movies - redundancy dataset\n" + "item meta rebuildable /mnt/plex/config /mnt/plex/config backup shallow\n" + "item big valuable /srv/big /srv/big backup\n" where := withData(t, composed, data) var mu sync.Mutex var walks []string truncated := true run := func(_ context.Context, name string, args ...string) (string, error) { mu.Lock() defer mu.Unlock() line := name + " " + strings.Join(args, " ") switch { case name == "test": return "", nil case name == "zfs" && args[len(args)-1] == "/storage/media/movies": return "storage/media\t97067690722560\n", nil case name == "zfs": return "", errors.New("not a ZFS dataset") case name == "bash" && strings.Contains(line, "-maxdepth 1"): return "1759744800 12673\n", nil case name == "bash": walks = append(walks, line) if strings.Contains(line, "-xdev") && !strings.Contains(line, "'/srv/big'") { t.Errorf("walked something declared not to be walked: %s", line) } if truncated { return fmt.Sprintf("123 1759700000 %d\n141\n", walkFiles), nil } return "999 1759700000 10\n0\n", nil } return "", nil } now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) b := &Backups{Where: where, Run: run, Now: func() time.Time { return now }, Say: quiet} must(t, b.MeasureAll(context.Background(), false)) got := b.Measurements()["media"] if m := got["movies"]; m.SizeBytes == nil || *m.SizeBytes != 97067690722560 || !strings.HasPrefix(m.Precision, "dataset storage/media") || m.LastWrite == nil { t.Fatalf("the library from its dataset: %+v", m) } if m := got["meta"]; m.SizeBytes != nil || !strings.HasPrefix(m.Precision, "no size") || m.LastWrite == nil { t.Fatalf("a shallow item: %+v", m) } if m := got["big"]; !strings.HasPrefix(m.Precision, "partial") { t.Fatalf("a walk stopped at its bound: %+v", m) } // An hour later the hourly round reads counters again and walks nothing. now = now.Add(time.Hour) truncated = false must(t, b.MeasureAll(context.Background(), false)) if len(walks) != 1 { t.Fatalf("walked %d times in two rounds an hour apart: %v", len(walks), walks) } // After the night, it walks. must(t, b.MeasureAll(context.Background(), true)) if len(walks) != 2 || b.Measurements()["media"]["big"].Precision != "exact" { t.Fatalf("%d walks, %+v", len(walks), b.Measurements()["media"]["big"]) } }