jochen
7524390cad
Walks run at most daily and stop after ten minutes or two million files; datasets are read from their counters and large items from their top level only. The database platform's tables are dumped with pg_dumpall rather than copied as live files.
306 lines
11 KiB
Go
306 lines
11 KiB
Go
package main
|
|
|
|
// The data the modules declare, measured, and a retired item deleted only after a last restore point
|
|
// (novox/hq ADR 0233) — over a fake restic and a fake machine.
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
const composedData = "# The data the modules on this machine declare, composed by the mesh. Do not edit.\n" +
|
|
"# postgres\nitem store irreplaceable /var/lib/mesh-store /var/lib/mesh-store/dumps\nitem dumps rebuildable /var/lib/mesh-store/dumps -\n" +
|
|
"# searxng\nitem valkey cache /var/lib/searxng/valkey-data -\n"
|
|
|
|
func withData(t *testing.T, declared, data string) Where {
|
|
t.Helper()
|
|
w := placed(t, declared)
|
|
w.Data = filepath.Join(w.State, "data.conf")
|
|
must(t, os.WriteFile(w.Data, []byte(data), 0o600))
|
|
return w
|
|
}
|
|
|
|
func TestTheComposedDataFileIsReadIntoEachModulesItems(t *testing.T) {
|
|
got, err := parseItems(composedData)
|
|
must(t, err)
|
|
if len(got["postgres"]) != 2 || got["postgres"][0].CoveredBy != "/var/lib/mesh-store/dumps" || got["postgres"][1].CoveredBy != "" ||
|
|
got["searxng"][0].Class != "cache" {
|
|
t.Fatalf("%+v", got)
|
|
}
|
|
for _, bad := range []string{"# pg\nitem a irreplaceable relative -\n", "# pg\nitem a irreplaceable /x\n", "# pg\nitem a b /x; rm -rf / -\n"} {
|
|
if _, err := parseItems(bad); err == nil || !strings.Contains(err.Error(), "pg declares a data line") {
|
|
t.Errorf("%q: %v", bad, err)
|
|
}
|
|
}
|
|
// An older controller composes no data file: nothing is measured, nothing fails.
|
|
b := &Backups{Where: placed(t, composed), Now: time.Now, Say: quiet}
|
|
if items, err := b.Items(); err != nil || len(items) != 0 {
|
|
t.Fatalf("%v, %v", items, err)
|
|
}
|
|
}
|
|
|
|
// Every item but a cache is measured — its files' size and its newest change, in one walk as root — and
|
|
// `backed-up` says each with the newest good night of the module that keeps the path covering it.
|
|
func TestEveryItemButACacheIsMeasuredAndSaidWithItsLastGoodBackup(t *testing.T) {
|
|
where := withData(t, composed, composedData)
|
|
var mu sync.Mutex
|
|
var walked []string
|
|
run := func(_ context.Context, name string, args ...string) (string, error) {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
switch {
|
|
case name == "test":
|
|
return "", nil
|
|
case name == "bash" && strings.Contains(args[1], "find '"):
|
|
walked = append(walked, args[1])
|
|
if strings.Contains(args[1], "'/var/lib/mesh-store'") {
|
|
return "1073741824 1759744800 4000\n0\n", nil
|
|
}
|
|
return "5000 1759700000 3\n0\n", nil
|
|
case name == "restic":
|
|
return "[]", nil
|
|
}
|
|
return "", nil
|
|
}
|
|
night := time.Date(2026, 10, 6, 3, 0, 0, 0, time.UTC)
|
|
b := &Backups{Where: where, Run: run, Now: func() time.Time { return night.Add(time.Hour) }, Say: quiet}
|
|
must(t, b.MeasureAll(context.Background(), true))
|
|
if len(walked) != 2 {
|
|
t.Fatalf("walked %d paths, want the two that are not a cache: %v", len(walked), walked)
|
|
}
|
|
b.recordGood("postgres", night)
|
|
got, err := b.BackedUp(context.Background(), "")
|
|
must(t, err)
|
|
var store, dumps *ItemReport
|
|
for _, m := range got {
|
|
for i := range m.Data {
|
|
switch m.Data[i].ID {
|
|
case "store":
|
|
store = &m.Data[i]
|
|
case "dumps":
|
|
dumps = &m.Data[i]
|
|
}
|
|
}
|
|
}
|
|
if store == nil || store.SizeBytes == nil || *store.SizeBytes != 1<<30 || store.LastWrite == nil ||
|
|
store.LastWrite.Unix() != 1759744800 || store.LastBackup == nil || !store.LastBackup.Equal(night) {
|
|
t.Fatalf("the store is said as %+v", store)
|
|
}
|
|
if dumps == nil || dumps.LastBackup != nil {
|
|
t.Fatalf("an item not backed up is said backed up: %+v", dumps)
|
|
}
|
|
}
|
|
|
|
// A retired item is deleted only after a last restore point of it, tagged as retired; a restore point
|
|
// that fails deletes nothing; and nothing declared now — itself, inside it, or holding it — is deleted.
|
|
func TestARetiredItemIsDeletedOnlyAfterALastRestorePoint(t *testing.T) {
|
|
where := withData(t, composed, composedData)
|
|
var mu sync.Mutex
|
|
var calls []string
|
|
failBackup := false
|
|
run := func(_ context.Context, name string, args ...string) (string, error) {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
line := name + " " + strings.Join(args, " ")
|
|
if name == "restic" {
|
|
line = "restic " + strings.Join(args[5:], " ")
|
|
}
|
|
calls = append(calls, line)
|
|
switch {
|
|
case name == "test":
|
|
return "", nil
|
|
case strings.HasPrefix(line, "restic backup"):
|
|
if failBackup {
|
|
return "", errors.New("the repository is locked")
|
|
}
|
|
return "{\"message_type\":\"summary\",\"snapshot_id\":\"0123456789abcdef\"}\n", nil
|
|
}
|
|
return "", nil
|
|
}
|
|
b := &Backups{Where: where, Run: run, Now: time.Now, Say: quiet}
|
|
ctx := context.Background()
|
|
for _, refused := range []struct{ path, want string }{
|
|
{"/var/lib/mesh-store", "declared now"},
|
|
{"/var/lib/mesh-store/dumps/old", "declared now"},
|
|
{"/var/lib", "declared now"},
|
|
{"/var/lib/mailu/data-mail", "backed up now"},
|
|
{"/srv", "too near the root"},
|
|
{"relative/x", "not a path"},
|
|
} {
|
|
if _, err := b.DeleteRetired(ctx, "m", "i", refused.path, "i", "op", "tidy"); err == nil || !strings.Contains(err.Error(), refused.want) {
|
|
t.Errorf("%s: %v, want %q", refused.path, err, refused.want)
|
|
}
|
|
}
|
|
if _, err := b.DeleteRetired(ctx, "house", "config", "/var/lib/house/config", "nope", "op", "tidy"); err == nil {
|
|
t.Error("a deletion without the item's name again was started")
|
|
}
|
|
if _, err := b.DeleteRetired(ctx, "house", "config", "/var/lib/house/config", "config", "op", ""); err == nil {
|
|
t.Error("a deletion without why was started")
|
|
}
|
|
|
|
wait := func(path string) Deletion {
|
|
t.Helper()
|
|
for i := 0; i < 200; i++ {
|
|
if d, err := b.DeletedOutcome(path); err == nil && d.Done {
|
|
return d
|
|
}
|
|
time.Sleep(5 * time.Millisecond)
|
|
}
|
|
t.Fatalf("the deletion of %s never finished", path)
|
|
return Deletion{}
|
|
}
|
|
|
|
mu.Lock()
|
|
failBackup = true
|
|
mu.Unlock()
|
|
_, err := b.DeleteRetired(ctx, "house", "config", "/var/lib/house/config", "config", "op", "moved to the anchor")
|
|
must(t, err)
|
|
if d := wait("/var/lib/house/config"); d.OK || !strings.Contains(d.Error, "nothing was deleted") {
|
|
t.Fatalf("a deletion with no restore point: %+v", d)
|
|
}
|
|
mu.Lock()
|
|
for _, c := range calls {
|
|
if strings.HasPrefix(c, "rm ") {
|
|
t.Fatal("it deleted without a last restore point")
|
|
}
|
|
}
|
|
mu.Unlock()
|
|
|
|
mu.Lock()
|
|
failBackup, calls = false, nil
|
|
mu.Unlock()
|
|
_, err = b.DeleteRetired(ctx, "house", "config", "/var/lib/house/config", "config", "op", "moved to the anchor")
|
|
must(t, err)
|
|
d := wait("/var/lib/house/config")
|
|
if !d.OK || d.Snapshot != "01234567" {
|
|
t.Fatalf("%+v", d)
|
|
}
|
|
var backup, rm = -1, -1
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
for i, c := range calls {
|
|
switch {
|
|
case c == "restic backup --json --tag module=house --tag retired=config /var/lib/house/config":
|
|
backup = i
|
|
case c == "rm -rf --one-file-system -- /var/lib/house/config":
|
|
rm = i
|
|
}
|
|
}
|
|
if backup < 0 || rm < 0 || rm < backup {
|
|
t.Fatalf("ran %v", calls)
|
|
}
|
|
// Asked again, it answers how it went rather than starting over.
|
|
again, err := b.DeleteRetired(ctx, "house", "config", "/var/lib/house/config", "config", "op", "moved")
|
|
if err != nil || !again.Done || !again.OK {
|
|
t.Fatalf("%+v, %v", again, err)
|
|
}
|
|
}
|
|
|
|
// The redundant storage under an item is read: a ZFS pool healthy, then degraded; an md array with a
|
|
// member missing; and plain storage, which is no redundancy at all.
|
|
func TestTheRedundantStorageUnderAnItemIsRead(t *testing.T) {
|
|
health, statusX := "ONLINE\n", "pool 'storage' is healthy\n"
|
|
fs := "storage/media zfs\n"
|
|
run := func(_ context.Context, name string, args ...string) (string, error) {
|
|
switch name {
|
|
case "findmnt":
|
|
return fs, nil
|
|
case "zpool":
|
|
if args[0] == "list" {
|
|
return health, nil
|
|
}
|
|
return statusX, nil
|
|
case "cat":
|
|
return "Personalities : [raid1]\nmd127 : active raid1 sdb1[1] sda1[0]\n 976630464 blocks super 1.2 [2/1] [U_]\n\nunused devices: <none>\n", nil
|
|
}
|
|
return "", nil
|
|
}
|
|
b := &Backups{Run: run, Now: time.Now, Say: quiet}
|
|
r := b.redundancyOf(context.Background(), "/storage/media/movies")
|
|
if r == nil || r.Kind != "zfs" || r.Where != "storage" || r.Healthy == nil || !*r.Healthy {
|
|
t.Fatalf("a healthy pool: %+v", r)
|
|
}
|
|
health, statusX = "DEGRADED\n", " pool: storage\n state: DEGRADED\nstatus: One or more devices has been removed\n"
|
|
if r := b.redundancyOf(context.Background(), "/storage/media/movies"); r.Healthy == nil || *r.Healthy || !strings.Contains(r.Said, "DEGRADED") {
|
|
t.Fatalf("a degraded pool: %+v", r)
|
|
}
|
|
fs = "/dev/md127 ext4\n"
|
|
if r := b.redundancyOf(context.Background(), "/srv/x"); r == nil || r.Kind != "md" || r.Healthy == nil || *r.Healthy {
|
|
t.Fatalf("an array missing a member: %+v", r)
|
|
}
|
|
fs = "/dev/nvme0n1p3 ext4\n"
|
|
if r := b.redundancyOf(context.Background(), "/var/lib/x"); r != nil {
|
|
t.Fatalf("plain storage read as redundant: %+v", r)
|
|
}
|
|
}
|
|
|
|
// Large items are never walked: a dataset is measured from the filesystem's counters and its top-level
|
|
// entries, a shallow item from its top-level entries only; a walk that meets more than its bound stops
|
|
// and says so; and a walk runs at most once a day — the hourly round keeps the last one.
|
|
func TestNothingLargeIsWalkedAndAWalkIsBoundedAndDaily(t *testing.T) {
|
|
data := "# media\nitem movies irreplaceable /storage/media/movies - redundancy dataset\n" +
|
|
"item meta rebuildable /mnt/plex/config /mnt/plex/config backup shallow\n" +
|
|
"item big valuable /srv/big /srv/big backup\n"
|
|
where := withData(t, composed, data)
|
|
var mu sync.Mutex
|
|
var walks []string
|
|
truncated := true
|
|
run := func(_ context.Context, name string, args ...string) (string, error) {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
line := name + " " + strings.Join(args, " ")
|
|
switch {
|
|
case name == "test":
|
|
return "", nil
|
|
case name == "zfs" && args[len(args)-1] == "/storage/media/movies":
|
|
return "storage/media\t97067690722560\n", nil
|
|
case name == "zfs":
|
|
return "", errors.New("not a ZFS dataset")
|
|
case name == "bash" && strings.Contains(line, "-maxdepth 1"):
|
|
return "1759744800 12673\n", nil
|
|
case name == "bash":
|
|
walks = append(walks, line)
|
|
if strings.Contains(line, "-xdev") && !strings.Contains(line, "'/srv/big'") {
|
|
t.Errorf("walked something declared not to be walked: %s", line)
|
|
}
|
|
if truncated {
|
|
return fmt.Sprintf("123 1759700000 %d\n141\n", walkFiles), nil
|
|
}
|
|
return "999 1759700000 10\n0\n", nil
|
|
}
|
|
return "", nil
|
|
}
|
|
now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
|
|
b := &Backups{Where: where, Run: run, Now: func() time.Time { return now }, Say: quiet}
|
|
must(t, b.MeasureAll(context.Background(), false))
|
|
got := b.Measurements()["media"]
|
|
if m := got["movies"]; m.SizeBytes == nil || *m.SizeBytes != 97067690722560 || !strings.HasPrefix(m.Precision, "dataset storage/media") || m.LastWrite == nil {
|
|
t.Fatalf("the library from its dataset: %+v", m)
|
|
}
|
|
if m := got["meta"]; m.SizeBytes != nil || !strings.HasPrefix(m.Precision, "no size") || m.LastWrite == nil {
|
|
t.Fatalf("a shallow item: %+v", m)
|
|
}
|
|
if m := got["big"]; !strings.HasPrefix(m.Precision, "partial") {
|
|
t.Fatalf("a walk stopped at its bound: %+v", m)
|
|
}
|
|
// An hour later the hourly round reads counters again and walks nothing.
|
|
now = now.Add(time.Hour)
|
|
truncated = false
|
|
must(t, b.MeasureAll(context.Background(), false))
|
|
if len(walks) != 1 {
|
|
t.Fatalf("walked %d times in two rounds an hour apart: %v", len(walks), walks)
|
|
}
|
|
// After the night, it walks.
|
|
must(t, b.MeasureAll(context.Background(), true))
|
|
if len(walks) != 2 || b.Measurements()["media"]["big"].Precision != "exact" {
|
|
t.Fatalf("%d walks, %+v", len(walks), b.Measurements()["media"]["big"])
|
|
}
|
|
}
|