jochen
685cb1cb1b
Backup lines are derived from each module's data section instead of written by hand; the holder measures declared items, reads the array under them, and deletes a retired item only after a last restore point; the Go providers say each held consumer's size so an empty replacement is seen.
242 lines
8.5 KiB
Go
242 lines
8.5 KiB
Go
package main
|
|
|
|
// The data the modules declare, measured, and a retired item deleted only after a last restore point
|
|
// (novox/hq ADR 0233) — over a fake restic and a fake machine.
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
const composedData = "# The data the modules on this machine declare, composed by the mesh. Do not edit.\n" +
|
|
"# postgres\nitem store irreplaceable /var/lib/mesh-store /var/lib/mesh-store/dumps\nitem dumps rebuildable /var/lib/mesh-store/dumps -\n" +
|
|
"# searxng\nitem valkey cache /var/lib/searxng/valkey-data -\n"
|
|
|
|
func withData(t *testing.T, declared, data string) Where {
|
|
t.Helper()
|
|
w := placed(t, declared)
|
|
w.Data = filepath.Join(w.State, "data.conf")
|
|
must(t, os.WriteFile(w.Data, []byte(data), 0o600))
|
|
return w
|
|
}
|
|
|
|
func TestTheComposedDataFileIsReadIntoEachModulesItems(t *testing.T) {
|
|
got, err := parseItems(composedData)
|
|
must(t, err)
|
|
if len(got["postgres"]) != 2 || got["postgres"][0].CoveredBy != "/var/lib/mesh-store/dumps" || got["postgres"][1].CoveredBy != "" ||
|
|
got["searxng"][0].Class != "cache" {
|
|
t.Fatalf("%+v", got)
|
|
}
|
|
for _, bad := range []string{"# pg\nitem a irreplaceable relative -\n", "# pg\nitem a irreplaceable /x\n", "# pg\nitem a b /x; rm -rf / -\n"} {
|
|
if _, err := parseItems(bad); err == nil || !strings.Contains(err.Error(), "pg declares a data line") {
|
|
t.Errorf("%q: %v", bad, err)
|
|
}
|
|
}
|
|
// An older controller composes no data file: nothing is measured, nothing fails.
|
|
b := &Backups{Where: placed(t, composed), Now: time.Now, Say: quiet}
|
|
if items, err := b.Items(); err != nil || len(items) != 0 {
|
|
t.Fatalf("%v, %v", items, err)
|
|
}
|
|
}
|
|
|
|
// Every item but a cache is measured — its files' size and its newest change, in one walk as root — and
|
|
// `backed-up` says each with the newest good night of the module that keeps the path covering it.
|
|
func TestEveryItemButACacheIsMeasuredAndSaidWithItsLastGoodBackup(t *testing.T) {
|
|
where := withData(t, composed, composedData)
|
|
var mu sync.Mutex
|
|
var walked []string
|
|
run := func(_ context.Context, name string, args ...string) (string, error) {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
switch {
|
|
case name == "test":
|
|
return "", nil
|
|
case name == "sh" && strings.Contains(args[1], "find '"):
|
|
walked = append(walked, args[1])
|
|
if strings.Contains(args[1], "'/var/lib/mesh-store'") {
|
|
return "1073741824 1759744800\n", nil
|
|
}
|
|
return "5000 1759700000\n", nil
|
|
case name == "restic":
|
|
return "[]", nil
|
|
}
|
|
return "", nil
|
|
}
|
|
night := time.Date(2026, 10, 6, 3, 0, 0, 0, time.UTC)
|
|
b := &Backups{Where: where, Run: run, Now: func() time.Time { return night.Add(time.Hour) }, Say: quiet}
|
|
must(t, b.MeasureAll(context.Background()))
|
|
if len(walked) != 2 {
|
|
t.Fatalf("walked %d paths, want the two that are not a cache: %v", len(walked), walked)
|
|
}
|
|
b.recordGood("postgres", night)
|
|
got, err := b.BackedUp(context.Background(), "")
|
|
must(t, err)
|
|
var store, dumps *ItemReport
|
|
for _, m := range got {
|
|
for i := range m.Data {
|
|
switch m.Data[i].ID {
|
|
case "store":
|
|
store = &m.Data[i]
|
|
case "dumps":
|
|
dumps = &m.Data[i]
|
|
}
|
|
}
|
|
}
|
|
if store == nil || store.SizeBytes == nil || *store.SizeBytes != 1<<30 || store.LastWrite == nil ||
|
|
store.LastWrite.Unix() != 1759744800 || store.LastBackup == nil || !store.LastBackup.Equal(night) {
|
|
t.Fatalf("the store is said as %+v", store)
|
|
}
|
|
if dumps == nil || dumps.LastBackup != nil {
|
|
t.Fatalf("an item not backed up is said backed up: %+v", dumps)
|
|
}
|
|
}
|
|
|
|
// A retired item is deleted only after a last restore point of it, tagged as retired; a restore point
|
|
// that fails deletes nothing; and nothing declared now — itself, inside it, or holding it — is deleted.
|
|
func TestARetiredItemIsDeletedOnlyAfterALastRestorePoint(t *testing.T) {
|
|
where := withData(t, composed, composedData)
|
|
var mu sync.Mutex
|
|
var calls []string
|
|
failBackup := false
|
|
run := func(_ context.Context, name string, args ...string) (string, error) {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
line := name + " " + strings.Join(args, " ")
|
|
if name == "restic" {
|
|
line = "restic " + strings.Join(args[5:], " ")
|
|
}
|
|
calls = append(calls, line)
|
|
switch {
|
|
case name == "test":
|
|
return "", nil
|
|
case strings.HasPrefix(line, "restic backup"):
|
|
if failBackup {
|
|
return "", errors.New("the repository is locked")
|
|
}
|
|
return "{\"message_type\":\"summary\",\"snapshot_id\":\"0123456789abcdef\"}\n", nil
|
|
}
|
|
return "", nil
|
|
}
|
|
b := &Backups{Where: where, Run: run, Now: time.Now, Say: quiet}
|
|
ctx := context.Background()
|
|
for _, refused := range []struct{ path, want string }{
|
|
{"/var/lib/mesh-store", "declared now"},
|
|
{"/var/lib/mesh-store/dumps/old", "declared now"},
|
|
{"/var/lib", "declared now"},
|
|
{"/var/lib/mailu/data-mail", "backed up now"},
|
|
{"/srv", "too near the root"},
|
|
{"relative/x", "not a path"},
|
|
} {
|
|
if _, err := b.DeleteRetired(ctx, "m", "i", refused.path, "i", "op", "tidy"); err == nil || !strings.Contains(err.Error(), refused.want) {
|
|
t.Errorf("%s: %v, want %q", refused.path, err, refused.want)
|
|
}
|
|
}
|
|
if _, err := b.DeleteRetired(ctx, "house", "config", "/var/lib/house/config", "nope", "op", "tidy"); err == nil {
|
|
t.Error("a deletion without the item's name again was started")
|
|
}
|
|
if _, err := b.DeleteRetired(ctx, "house", "config", "/var/lib/house/config", "config", "op", ""); err == nil {
|
|
t.Error("a deletion without why was started")
|
|
}
|
|
|
|
wait := func(path string) Deletion {
|
|
t.Helper()
|
|
for i := 0; i < 200; i++ {
|
|
if d, err := b.DeletedOutcome(path); err == nil && d.Done {
|
|
return d
|
|
}
|
|
time.Sleep(5 * time.Millisecond)
|
|
}
|
|
t.Fatalf("the deletion of %s never finished", path)
|
|
return Deletion{}
|
|
}
|
|
|
|
mu.Lock()
|
|
failBackup = true
|
|
mu.Unlock()
|
|
_, err := b.DeleteRetired(ctx, "house", "config", "/var/lib/house/config", "config", "op", "moved to the anchor")
|
|
must(t, err)
|
|
if d := wait("/var/lib/house/config"); d.OK || !strings.Contains(d.Error, "nothing was deleted") {
|
|
t.Fatalf("a deletion with no restore point: %+v", d)
|
|
}
|
|
mu.Lock()
|
|
for _, c := range calls {
|
|
if strings.HasPrefix(c, "rm ") {
|
|
t.Fatal("it deleted without a last restore point")
|
|
}
|
|
}
|
|
mu.Unlock()
|
|
|
|
mu.Lock()
|
|
failBackup, calls = false, nil
|
|
mu.Unlock()
|
|
_, err = b.DeleteRetired(ctx, "house", "config", "/var/lib/house/config", "config", "op", "moved to the anchor")
|
|
must(t, err)
|
|
d := wait("/var/lib/house/config")
|
|
if !d.OK || d.Snapshot != "01234567" {
|
|
t.Fatalf("%+v", d)
|
|
}
|
|
var backup, rm = -1, -1
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
for i, c := range calls {
|
|
switch {
|
|
case c == "restic backup --json --tag module=house --tag retired=config /var/lib/house/config":
|
|
backup = i
|
|
case c == "rm -rf --one-file-system -- /var/lib/house/config":
|
|
rm = i
|
|
}
|
|
}
|
|
if backup < 0 || rm < 0 || rm < backup {
|
|
t.Fatalf("ran %v", calls)
|
|
}
|
|
// Asked again, it answers how it went rather than starting over.
|
|
again, err := b.DeleteRetired(ctx, "house", "config", "/var/lib/house/config", "config", "op", "moved")
|
|
if err != nil || !again.Done || !again.OK {
|
|
t.Fatalf("%+v, %v", again, err)
|
|
}
|
|
}
|
|
|
|
// The redundant storage under an item is read: a ZFS pool healthy, then degraded; an md array with a
|
|
// member missing; and plain storage, which is no redundancy at all.
|
|
func TestTheRedundantStorageUnderAnItemIsRead(t *testing.T) {
|
|
health, statusX := "ONLINE\n", "pool 'storage' is healthy\n"
|
|
fs := "storage/media zfs\n"
|
|
run := func(_ context.Context, name string, args ...string) (string, error) {
|
|
switch name {
|
|
case "findmnt":
|
|
return fs, nil
|
|
case "zpool":
|
|
if args[0] == "list" {
|
|
return health, nil
|
|
}
|
|
return statusX, nil
|
|
case "cat":
|
|
return "Personalities : [raid1]\nmd127 : active raid1 sdb1[1] sda1[0]\n 976630464 blocks super 1.2 [2/1] [U_]\n\nunused devices: <none>\n", nil
|
|
}
|
|
return "", nil
|
|
}
|
|
b := &Backups{Run: run, Now: time.Now, Say: quiet}
|
|
r := b.redundancyOf(context.Background(), "/storage/media/movies")
|
|
if r == nil || r.Kind != "zfs" || r.Where != "storage" || r.Healthy == nil || !*r.Healthy {
|
|
t.Fatalf("a healthy pool: %+v", r)
|
|
}
|
|
health, statusX = "DEGRADED\n", " pool: storage\n state: DEGRADED\nstatus: One or more devices has been removed\n"
|
|
if r := b.redundancyOf(context.Background(), "/storage/media/movies"); r.Healthy == nil || *r.Healthy || !strings.Contains(r.Said, "DEGRADED") {
|
|
t.Fatalf("a degraded pool: %+v", r)
|
|
}
|
|
fs = "/dev/md127 ext4\n"
|
|
if r := b.redundancyOf(context.Background(), "/srv/x"); r == nil || r.Kind != "md" || r.Healthy == nil || *r.Healthy {
|
|
t.Fatalf("an array missing a member: %+v", r)
|
|
}
|
|
fs = "/dev/nvme0n1p3 ext4\n"
|
|
if r := b.redundancyOf(context.Background(), "/var/lib/x"); r != nil {
|
|
t.Fatalf("plain storage read as redundant: %+v", r)
|
|
}
|
|
}
|