diff --git a/modules/distribution/module.json b/modules/distribution/module.json index 06446f7..e8eb32d 100644 --- a/modules/distribution/module.json +++ b/modules/distribution/module.json @@ -43,6 +43,7 @@ "path": "${dir:registry-data}", "class": "rebuildable", "backup": "none", + "measure": "shallow", "why": "the artifact store: every image is built again from its source, and too large to copy every night (ADR 0214 left it out)" } ] diff --git a/modules/ollama/module.json b/modules/ollama/module.json index 12bad0a..2291c63 100644 --- a/modules/ollama/module.json +++ b/modules/ollama/module.json @@ -32,6 +32,7 @@ "path": "${dir:state}", "class": "rebuildable", "backup": "none", + "measure": "shallow", "why": "models, downloaded again, and too large to copy every night" } ] diff --git a/modules/restic/cmd/restic-backups/data.go b/modules/restic/cmd/restic-backups/data.go index 429eba4..f0d6dd5 100644 --- a/modules/restic/cmd/restic-backups/data.go +++ b/modules/restic/cmd/restic-backups/data.go @@ -3,13 +3,23 @@ // The mesh composes a second file beside the backup lines: every data item every module on the // machine declares, one line each, // -// item +// item // // where covered-by is the path whose snapshot keeps it (the item itself, or the directory its dump -// writes into), or `-` when it is not backed up, and protection is backup, redundancy, both, or none. -// This holder measures each item that is not a cache — its size, its newest write, and the redundant -// storage it is on and whether that is healthy (ZFS, md, btrfs) — once an hour, and says it with each -// module's last good backup in `backed-up`. The controller keeps the readings and says when an item shrinks, stops being written, +// writes into), or `-` when it is not backed up; protection is backup, redundancy, both, or none; and +// measure is how the item is measured. This holder measures each item that is not a cache — its size, +// its newest write, and the redundant storage it is on and whether that is healthy (ZFS, md, btrfs) — +// and says it, with each module's last good backup and the precision of the measurement, in +// `backed-up`. +// +// **Never an unbounded walk of something large.** Three methods, the item's own choice: +// +// - `walk` (the default, for small items): every file summed and the newest change found, in one walk +// as root — at most once a day, and stopped after walkFor or walkFiles, said as "measured partially"; +// - `dataset`: the ZFS dataset holding the path — its `used` from the filesystem's own counters — and +// the newest change among the path's top-level entries; hourly, and nothing is walked; +// - `shallow`: the newest change among the top-level entries only, and no size; hourly. The controller keeps the readings and says when an item shrinks, stops being written, +// // goes without a backup, or is replaced by an empty copy of itself; this holder only measures. // // And it deletes, when a person has decided: an item the mesh retired — its module gone from this @@ -33,6 +43,8 @@ import ( // Item is one data item a module declares. type Item struct { + // Measure is walk, dataset or shallow. + Measure string `json:"measure,omitempty"` Module string `json:"-"` ID string `json:"item"` Class string `json:"class"` @@ -54,9 +66,12 @@ type Redundancy struct { // Measured is what one measurement found. type Measured struct { - SizeBytes *int64 `json:"size_bytes,omitempty"` - LastWrite *time.Time `json:"last_write,omitempty"` - MeasuredAt *time.Time `json:"measured_at,omitempty"` + SizeBytes *int64 `json:"size_bytes,omitempty"` + LastWrite *time.Time `json:"last_write,omitempty"` + MeasuredAt *time.Time `json:"measured_at,omitempty"` + // Precision says what the size is: "exact", "dataset : its whole size", "partial: …" (a lower + // bound, never compared), or "no size: …". + Precision string `json:"precision,omitempty"` Error string `json:"error,omitempty"` Redundancy *Redundancy `json:"redundancy,omitempty"` } @@ -91,13 +106,22 @@ func parseItems(text string) (map[string][]Item, error) { continue } f := strings.Fields(line) - if (len(f) != 5 && len(f) != 6) || f[0] != "item" || !safePath.MatchString(f[3]) || (f[4] != "-" && !safePath.MatchString(f[4])) { + if len(f) < 5 || len(f) > 7 || f[0] != "item" || !safePath.MatchString(f[3]) || (f[4] != "-" && !safePath.MatchString(f[4])) { return nil, fmt.Errorf("%s declares a data line this holder does not read: %s", module, line) } it := Item{Module: module, ID: f[1], Class: f[2], Path: f[3]} - if len(f) == 6 { + if len(f) >= 6 { it.Protection = f[5] } + it.Measure = measureWalk + if len(f) == 7 { + it.Measure = f[6] + } + switch it.Measure { + case measureWalk, measureDataset, measureShallow: + default: + return nil, fmt.Errorf("%s declares a data line this holder does not read: %s", module, line) + } if f[4] != "-" { it.CoveredBy = f[4] } @@ -135,45 +159,105 @@ func (b *Backups) Measurements() map[string]map[string]Measured { var measuring sync.Mutex -// measureCommand sums the files under a path and finds its newest change, in one walk, as root: a -// store's directory is often its own user's alone. One line out: " ". -func measureCommand(path string) string { - return "find '" + path + "' -xdev -printf '%y %s %T@\\n' 2>/dev/null | " + - "awk 'BEGIN{s=0;m=0} {if ($1==\"f\") s+=$2; if ($3>m) m=$3} END {printf \"%d %.0f\\n\", s, m}'" +// The ways an item is measured. +const ( + measureWalk = "walk" + measureDataset = "dataset" + measureShallow = "shallow" +) + +// The bounds on a walk, and how often one runs: a walk is for small items, and one that meets a large +// one stops and says so rather than loading the disks for hours. +var ( + walkFor = 10 * time.Minute + walkFiles = 2_000_000 + walkEvery = 23 * time.Hour +) + +// walkCommand sums the files under a path and finds its newest change, in one walk, as root — bounded +// by time and by files. One line out: " ". +func walkCommand(path string) string { + return fmt.Sprintf("timeout %d find '%s' -xdev -printf '%%y %%s %%T@\\n' 2>/dev/null | head -n %d | "+ + "awk 'BEGIN{s=0;m=0;n=0} {n++; if ($1==\"f\") s+=$2; if ($3>m) m=$3} END {printf \"%%d %%.0f %%d\\n\", s, m, n}'; "+ + "echo \"${PIPESTATUS[0]:-0}\"", int(walkFor.Seconds()), path, walkFiles) } -// measure is one item, measured now. +// topCommand is the newest change among a path and its top-level entries, and how many there are: +// one directory read, bounded. " ". +func topCommand(path string) string { + return "timeout 60 find '" + path + "' -maxdepth 1 -printf '%T@\\n' 2>/dev/null | " + + "awk 'BEGIN{m=0;n=0} {n++; if ($1>m) m=$1} END {printf \"%.0f %d\\n\", m, n-1}'" +} + +func epochTime(s string) *time.Time { + epoch, err := strconv.ParseInt(s, 10, 64) + if err != nil || epoch <= 0 { + return nil + } + t := time.Unix(epoch, 0).UTC() + return &t +} + +// measure is one item, measured now, by its own method. func (b *Backups) measure(ctx context.Context, it Item) Measured { at := b.Now().UTC() m := Measured{MeasuredAt: &at} if !b.exists(ctx, it.Path) { m.Error = it.Path + " does not exist" zero := int64(0) - m.SizeBytes = &zero + m.SizeBytes, m.Precision = &zero, "exact" return m } - out, err := b.Run(ctx, "sh", "-c", measureCommand(it.Path)) + m.Redundancy = b.redundancyOf(ctx, it.Path) + switch it.Measure { + case measureDataset, measureShallow: + out, err := b.Run(ctx, "bash", "-c", topCommand(it.Path)) + if f := strings.Fields(out); err == nil && len(f) == 2 { + m.LastWrite = epochTime(f[0]) + } else if err != nil { + m.Error = err.Error() + } + m.Precision = "no size: the newest change among its top-level entries only" + if it.Measure == measureShallow { + return m + } + out, err = b.Run(ctx, "zfs", "list", "-Hp", "-o", "name,used", it.Path) + f := strings.Fields(out) + if err != nil || len(f) != 2 { + m.Precision = "no size: it is on no ZFS dataset to read one from; the newest change among its top-level entries only" + return m + } + used, err := strconv.ParseInt(f[1], 10, 64) + if err != nil { + return m + } + m.SizeBytes = &used + m.Precision = "dataset " + f[0] + ": its whole size, from the filesystem's counters; the newest change among the top-level entries" + return m + } + out, err := b.Run(ctx, "bash", "-c", walkCommand(it.Path)) if err != nil { m.Error = err.Error() return m } - f := strings.Fields(out) - if len(f) != 2 { + lines := strings.Split(strings.TrimSpace(out), "\n") + f := strings.Fields(lines[0]) + if len(f) != 3 || len(lines) < 2 { m.Error = "the measurement said " + strings.TrimSpace(out) return m } size, err1 := strconv.ParseInt(f[0], 10, 64) - epoch, err2 := strconv.ParseInt(f[1], 10, 64) + files, err2 := strconv.Atoi(f[2]) if err1 != nil || err2 != nil { m.Error = "the measurement said " + strings.TrimSpace(out) return m } - m.SizeBytes = &size - if epoch > 0 { - w := time.Unix(epoch, 0).UTC() - m.LastWrite = &w + m.SizeBytes, m.LastWrite = &size, epochTime(f[1]) + m.Precision = "exact" + if status := strings.TrimSpace(lines[len(lines)-1]); status == "124" || files >= walkFiles { + m.Precision = fmt.Sprintf("partial: measured partially — stopped after %s or %d files; the size is a lower bound, "+ + "and this item wants measure: dataset or shallow", walkFor, walkFiles) } - m.Redundancy = b.redundancyOf(ctx, it.Path) return m } @@ -276,14 +360,17 @@ func firstLineOf(s string) string { func oneLineOf(s string) string { return strings.Join(strings.Fields(s), " ") } -// MeasureAll measures every item that is not a cache, one at a time, and keeps what it found. -func (b *Backups) MeasureAll(ctx context.Context) error { +// MeasureAll measures every item that is not a cache, one at a time, and keeps what it found. An item +// measured by a walk is walked only when walks is true or its last walk is older than walkEvery: the +// hourly round reads counters, and a walk runs at most once a day. +func (b *Backups) MeasureAll(ctx context.Context, walks bool) error { measuring.Lock() defer measuring.Unlock() items, err := b.Items() if err != nil { return err } + before := b.Measurements() found := map[string]map[string]Measured{} for module, its := range items { for _, it := range its { @@ -293,6 +380,11 @@ func (b *Backups) MeasureAll(ctx context.Context) error { if found[module] == nil { found[module] = map[string]Measured{} } + if was, ok := before[module][it.ID]; ok && it.Measure == measureWalk && !walks && was.MeasuredAt != nil && + b.Now().Sub(*was.MeasuredAt) < walkEvery { + found[module][it.ID] = was + continue + } found[module][it.ID] = b.measure(ctx, it) } } diff --git a/modules/restic/cmd/restic-backups/data_test.go b/modules/restic/cmd/restic-backups/data_test.go index 79582b3..0834ad6 100644 --- a/modules/restic/cmd/restic-backups/data_test.go +++ b/modules/restic/cmd/restic-backups/data_test.go @@ -6,6 +6,7 @@ package main import ( "context" "errors" + "fmt" "os" "path/filepath" "strings" @@ -57,12 +58,12 @@ func TestEveryItemButACacheIsMeasuredAndSaidWithItsLastGoodBackup(t *testing.T) switch { case name == "test": return "", nil - case name == "sh" && strings.Contains(args[1], "find '"): + case name == "bash" && strings.Contains(args[1], "find '"): walked = append(walked, args[1]) if strings.Contains(args[1], "'/var/lib/mesh-store'") { - return "1073741824 1759744800\n", nil + return "1073741824 1759744800 4000\n0\n", nil } - return "5000 1759700000\n", nil + return "5000 1759700000 3\n0\n", nil case name == "restic": return "[]", nil } @@ -70,7 +71,7 @@ func TestEveryItemButACacheIsMeasuredAndSaidWithItsLastGoodBackup(t *testing.T) } night := time.Date(2026, 10, 6, 3, 0, 0, 0, time.UTC) b := &Backups{Where: where, Run: run, Now: func() time.Time { return night.Add(time.Hour) }, Say: quiet} - must(t, b.MeasureAll(context.Background())) + must(t, b.MeasureAll(context.Background(), true)) if len(walked) != 2 { t.Fatalf("walked %d paths, want the two that are not a cache: %v", len(walked), walked) } @@ -239,3 +240,66 @@ func TestTheRedundantStorageUnderAnItemIsRead(t *testing.T) { t.Fatalf("plain storage read as redundant: %+v", r) } } + +// Large items are never walked: a dataset is measured from the filesystem's counters and its top-level +// entries, a shallow item from its top-level entries only; a walk that meets more than its bound stops +// and says so; and a walk runs at most once a day — the hourly round keeps the last one. +func TestNothingLargeIsWalkedAndAWalkIsBoundedAndDaily(t *testing.T) { + data := "# media\nitem movies irreplaceable /storage/media/movies - redundancy dataset\n" + + "item meta rebuildable /mnt/plex/config /mnt/plex/config backup shallow\n" + + "item big valuable /srv/big /srv/big backup\n" + where := withData(t, composed, data) + var mu sync.Mutex + var walks []string + truncated := true + run := func(_ context.Context, name string, args ...string) (string, error) { + mu.Lock() + defer mu.Unlock() + line := name + " " + strings.Join(args, " ") + switch { + case name == "test": + return "", nil + case name == "zfs" && args[len(args)-1] == "/storage/media/movies": + return "storage/media\t97067690722560\n", nil + case name == "zfs": + return "", errors.New("not a ZFS dataset") + case name == "bash" && strings.Contains(line, "-maxdepth 1"): + return "1759744800 12673\n", nil + case name == "bash": + walks = append(walks, line) + if strings.Contains(line, "-xdev") && !strings.Contains(line, "'/srv/big'") { + t.Errorf("walked something declared not to be walked: %s", line) + } + if truncated { + return fmt.Sprintf("123 1759700000 %d\n141\n", walkFiles), nil + } + return "999 1759700000 10\n0\n", nil + } + return "", nil + } + now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) + b := &Backups{Where: where, Run: run, Now: func() time.Time { return now }, Say: quiet} + must(t, b.MeasureAll(context.Background(), false)) + got := b.Measurements()["media"] + if m := got["movies"]; m.SizeBytes == nil || *m.SizeBytes != 97067690722560 || !strings.HasPrefix(m.Precision, "dataset storage/media") || m.LastWrite == nil { + t.Fatalf("the library from its dataset: %+v", m) + } + if m := got["meta"]; m.SizeBytes != nil || !strings.HasPrefix(m.Precision, "no size") || m.LastWrite == nil { + t.Fatalf("a shallow item: %+v", m) + } + if m := got["big"]; !strings.HasPrefix(m.Precision, "partial") { + t.Fatalf("a walk stopped at its bound: %+v", m) + } + // An hour later the hourly round reads counters again and walks nothing. + now = now.Add(time.Hour) + truncated = false + must(t, b.MeasureAll(context.Background(), false)) + if len(walks) != 1 { + t.Fatalf("walked %d times in two rounds an hour apart: %v", len(walks), walks) + } + // After the night, it walks. + must(t, b.MeasureAll(context.Background(), true)) + if len(walks) != 2 || b.Measurements()["media"]["big"].Precision != "exact" { + t.Fatalf("%d walks, %+v", len(walks), b.Measurements()["media"]["big"]) + } +} diff --git a/modules/restic/cmd/restic-backups/main.go b/modules/restic/cmd/restic-backups/main.go index 2e31bb9..1e53e18 100644 --- a/modules/restic/cmd/restic-backups/main.go +++ b/modules/restic/cmd/restic-backups/main.go @@ -58,7 +58,8 @@ func nights(b *Backups) { } // measurements measures every declared item once an hour (MESH_BACKUP_MEASURE_EVERY to change it), -// the first a few minutes after start, and after every night (novox/hq ADR 0233). +// the first a few minutes after start — counters hourly, a walk at most daily — and every walk after +// each night (novox/hq ADR 0233). func measurements(b *Backups) { every := time.Hour if d, err := time.ParseDuration(os.Getenv("MESH_BACKUP_MEASURE_EVERY")); err == nil && d >= time.Minute { @@ -66,7 +67,7 @@ func measurements(b *Backups) { } time.Sleep(3 * time.Minute) for { - if err := b.MeasureAll(context.Background()); err != nil { + if err := b.MeasureAll(context.Background(), false); err != nil { say("measuring the data declared here failed: %v", err) } time.Sleep(every) @@ -89,7 +90,7 @@ func night(b *Backups) { if len(failed) > 0 { say("the night left %s without a backup", strings.Join(failed, ", ")) } - if err := b.MeasureAll(ctx); err != nil { + if err := b.MeasureAll(ctx, true); err != nil { say("measuring the data declared here failed: %v", err) } // Sundays, the repository's own integrity with a sample of the data read back. diff --git a/modules/supabase/module.json b/modules/supabase/module.json index 0934674..0fcb70d 100644 --- a/modules/supabase/module.json +++ b/modules/supabase/module.json @@ -64,7 +64,17 @@ "id": "db", "path": "${dir:db-data}", "class": "valuable", - "why": "every table; copied as live files, which may not restore \u2014 a dump is wanted" + "backup": { + "dump": "docker exec supabase-db pg_dumpall -h 127.0.0.1 -U supabase_admin > ${dir:dumps}/all.sql.partial && mv ${dir:dumps}/all.sql.partial ${dir:dumps}/all.sql", + "into": "dumps" + }, + "why": "every table; copied by pg_dumpall inside the database's container (local connections are trusted there), since a running store's files are not a consistent copy" + }, + { + "id": "dumps", + "path": "${dir:dumps}", + "class": "rebuildable", + "why": "last night's dump of the database, made again every night" }, { "id": "storage", @@ -111,6 +121,12 @@ "mode": "0700", "owner": "105:106" }, + { + "id": "dumps", + "type": "directory", + "path": "${dir:state}/dumps", + "mode": "0700" + }, { "id": "db-config", "type": "directory",