Bound the holder's measuring; dump the database platform (hq ADR 0233)

Walks run at most daily and stop after ten minutes or two million files; datasets are read from
their counters and large items from their top level only. The database platform's tables are
dumped with pg_dumpall rather than copied as live files.

modules/distribution/module.json

@@ -43,6 +43,7 @@
         "path": "${dir:registry-data}",
         "class": "rebuildable",
         "backup": "none",
+        "measure": "shallow",
         "why": "the artifact store: every image is built again from its source, and too large to copy every night (ADR 0214 left it out)"
       }
     ]

modules/ollama/module.json

@@ -32,6 +32,7 @@
         "path": "${dir:state}",
         "class": "rebuildable",
         "backup": "none",
+        "measure": "shallow",
         "why": "models, downloaded again, and too large to copy every night"
       }
     ]

modules/restic/cmd/restic-backups/data.go

@@ -3,13 +3,23 @@
 // The mesh composes a second file beside the backup lines: every data item every module on the
 // machine declares, one line each,
 //
-//	item <id> <class> <path> <covered-by> <protection>
+//	item <id> <class> <path> <covered-by> <protection> <measure>
 //
 // where covered-by is the path whose snapshot keeps it (the item itself, or the directory its dump
-// writes into), or `-` when it is not backed up, and protection is backup, redundancy, both, or none.
-// This holder measures each item that is not a cache — its size, its newest write, and the redundant
-// storage it is on and whether that is healthy (ZFS, md, btrfs) — once an hour, and says it with each
-// module's last good backup in `backed-up`. The controller keeps the readings and says when an item shrinks, stops being written,
+// writes into), or `-` when it is not backed up; protection is backup, redundancy, both, or none; and
+// measure is how the item is measured. This holder measures each item that is not a cache — its size,
+// its newest write, and the redundant storage it is on and whether that is healthy (ZFS, md, btrfs) —
+// and says it, with each module's last good backup and the precision of the measurement, in
+// `backed-up`.
+//
+// **Never an unbounded walk of something large.** Three methods, the item's own choice:
+//
+//   - `walk` (the default, for small items): every file summed and the newest change found, in one walk
+//     as root — at most once a day, and stopped after walkFor or walkFiles, said as "measured partially";
+//   - `dataset`: the ZFS dataset holding the path — its `used` from the filesystem's own counters — and
+//     the newest change among the path's top-level entries; hourly, and nothing is walked;
+//   - `shallow`: the newest change among the top-level entries only, and no size; hourly. The controller keeps the readings and says when an item shrinks, stops being written,
+//
 // goes without a backup, or is replaced by an empty copy of itself; this holder only measures.
 //
 // And it deletes, when a person has decided: an item the mesh retired — its module gone from this
@@ -33,6 +43,8 @@ import (
 
 // Item is one data item a module declares.
 type Item struct {
+	// Measure is walk, dataset or shallow.
+	Measure    string `json:"measure,omitempty"`
 	Module     string `json:"-"`
 	ID         string `json:"item"`
 	Class      string `json:"class"`
@@ -54,9 +66,12 @@ type Redundancy struct {
 
 // Measured is what one measurement found.
 type Measured struct {
-	SizeBytes  *int64      `json:"size_bytes,omitempty"`
-	LastWrite  *time.Time  `json:"last_write,omitempty"`
-	MeasuredAt *time.Time  `json:"measured_at,omitempty"`
+	SizeBytes  *int64     `json:"size_bytes,omitempty"`
+	LastWrite  *time.Time `json:"last_write,omitempty"`
+	MeasuredAt *time.Time `json:"measured_at,omitempty"`
+	// Precision says what the size is: "exact", "dataset <name>: its whole size", "partial: …" (a lower
+	// bound, never compared), or "no size: …".
+	Precision  string      `json:"precision,omitempty"`
 	Error      string      `json:"error,omitempty"`
 	Redundancy *Redundancy `json:"redundancy,omitempty"`
 }
@@ -91,13 +106,22 @@ func parseItems(text string) (map[string][]Item, error) {
 			continue
 		}
 		f := strings.Fields(line)
-		if (len(f) != 5 && len(f) != 6) || f[0] != "item" || !safePath.MatchString(f[3]) || (f[4] != "-" && !safePath.MatchString(f[4])) {
+		if len(f) < 5 || len(f) > 7 || f[0] != "item" || !safePath.MatchString(f[3]) || (f[4] != "-" && !safePath.MatchString(f[4])) {
 			return nil, fmt.Errorf("%s declares a data line this holder does not read: %s", module, line)
 		}
 		it := Item{Module: module, ID: f[1], Class: f[2], Path: f[3]}
-		if len(f) == 6 {
+		if len(f) >= 6 {
 			it.Protection = f[5]
 		}
+		it.Measure = measureWalk
+		if len(f) == 7 {
+			it.Measure = f[6]
+		}
+		switch it.Measure {
+		case measureWalk, measureDataset, measureShallow:
+		default:
+			return nil, fmt.Errorf("%s declares a data line this holder does not read: %s", module, line)
+		}
 		if f[4] != "-" {
 			it.CoveredBy = f[4]
 		}
@@ -135,45 +159,105 @@ func (b *Backups) Measurements() map[string]map[string]Measured {
 
 var measuring sync.Mutex
 
-// measureCommand sums the files under a path and finds its newest change, in one walk, as root: a
-// store's directory is often its own user's alone. One line out: "<bytes> <epoch seconds>".
-func measureCommand(path string) string {
-	return "find '" + path + "' -xdev -printf '%y %s %T@\\n' 2>/dev/null | " +
-		"awk 'BEGIN{s=0;m=0} {if ($1==\"f\") s+=$2; if ($3>m) m=$3} END {printf \"%d %.0f\\n\", s, m}'"
+// The ways an item is measured.
+const (
+	measureWalk    = "walk"
+	measureDataset = "dataset"
+	measureShallow = "shallow"
+)
+
+// The bounds on a walk, and how often one runs: a walk is for small items, and one that meets a large
+// one stops and says so rather than loading the disks for hours.
+var (
+	walkFor   = 10 * time.Minute
+	walkFiles = 2_000_000
+	walkEvery = 23 * time.Hour
+)
+
+// walkCommand sums the files under a path and finds its newest change, in one walk, as root — bounded
+// by time and by files. One line out: "<bytes> <epoch seconds> <files> <complete 1|0>".
+func walkCommand(path string) string {
+	return fmt.Sprintf("timeout %d find '%s' -xdev -printf '%%y %%s %%T@\\n' 2>/dev/null | head -n %d | "+
+		"awk 'BEGIN{s=0;m=0;n=0} {n++; if ($1==\"f\") s+=$2; if ($3>m) m=$3} END {printf \"%%d %%.0f %%d\\n\", s, m, n}'; "+
+		"echo \"${PIPESTATUS[0]:-0}\"", int(walkFor.Seconds()), path, walkFiles)
 }
 
-// measure is one item, measured now.
+// topCommand is the newest change among a path and its top-level entries, and how many there are:
+// one directory read, bounded. "<epoch seconds> <entries>".
+func topCommand(path string) string {
+	return "timeout 60 find '" + path + "' -maxdepth 1 -printf '%T@\\n' 2>/dev/null | " +
+		"awk 'BEGIN{m=0;n=0} {n++; if ($1>m) m=$1} END {printf \"%.0f %d\\n\", m, n-1}'"
+}
+
+func epochTime(s string) *time.Time {
+	epoch, err := strconv.ParseInt(s, 10, 64)
+	if err != nil || epoch <= 0 {
+		return nil
+	}
+	t := time.Unix(epoch, 0).UTC()
+	return &t
+}
+
+// measure is one item, measured now, by its own method.
 func (b *Backups) measure(ctx context.Context, it Item) Measured {
 	at := b.Now().UTC()
 	m := Measured{MeasuredAt: &at}
 	if !b.exists(ctx, it.Path) {
 		m.Error = it.Path + " does not exist"
 		zero := int64(0)
-		m.SizeBytes = &zero
+		m.SizeBytes, m.Precision = &zero, "exact"
 		return m
 	}
-	out, err := b.Run(ctx, "sh", "-c", measureCommand(it.Path))
+	m.Redundancy = b.redundancyOf(ctx, it.Path)
+	switch it.Measure {
+	case measureDataset, measureShallow:
+		out, err := b.Run(ctx, "bash", "-c", topCommand(it.Path))
+		if f := strings.Fields(out); err == nil && len(f) == 2 {
+			m.LastWrite = epochTime(f[0])
+		} else if err != nil {
+			m.Error = err.Error()
+		}
+		m.Precision = "no size: the newest change among its top-level entries only"
+		if it.Measure == measureShallow {
+			return m
+		}
+		out, err = b.Run(ctx, "zfs", "list", "-Hp", "-o", "name,used", it.Path)
+		f := strings.Fields(out)
+		if err != nil || len(f) != 2 {
+			m.Precision = "no size: it is on no ZFS dataset to read one from; the newest change among its top-level entries only"
+			return m
+		}
+		used, err := strconv.ParseInt(f[1], 10, 64)
+		if err != nil {
+			return m
+		}
+		m.SizeBytes = &used
+		m.Precision = "dataset " + f[0] + ": its whole size, from the filesystem's counters; the newest change among the top-level entries"
+		return m
+	}
+	out, err := b.Run(ctx, "bash", "-c", walkCommand(it.Path))
 	if err != nil {
 		m.Error = err.Error()
 		return m
 	}
-	f := strings.Fields(out)
-	if len(f) != 2 {
+	lines := strings.Split(strings.TrimSpace(out), "\n")
+	f := strings.Fields(lines[0])
+	if len(f) != 3 || len(lines) < 2 {
 		m.Error = "the measurement said " + strings.TrimSpace(out)
 		return m
 	}
 	size, err1 := strconv.ParseInt(f[0], 10, 64)
-	epoch, err2 := strconv.ParseInt(f[1], 10, 64)
+	files, err2 := strconv.Atoi(f[2])
 	if err1 != nil || err2 != nil {
 		m.Error = "the measurement said " + strings.TrimSpace(out)
 		return m
 	}
-	m.SizeBytes = &size
-	if epoch > 0 {
-		w := time.Unix(epoch, 0).UTC()
-		m.LastWrite = &w
+	m.SizeBytes, m.LastWrite = &size, epochTime(f[1])
+	m.Precision = "exact"
+	if status := strings.TrimSpace(lines[len(lines)-1]); status == "124" || files >= walkFiles {
+		m.Precision = fmt.Sprintf("partial: measured partially — stopped after %s or %d files; the size is a lower bound, "+
+			"and this item wants measure: dataset or shallow", walkFor, walkFiles)
 	}
-	m.Redundancy = b.redundancyOf(ctx, it.Path)
 	return m
 }
 
@@ -276,14 +360,17 @@ func firstLineOf(s string) string {
 
 func oneLineOf(s string) string { return strings.Join(strings.Fields(s), " ") }
 
-// MeasureAll measures every item that is not a cache, one at a time, and keeps what it found.
-func (b *Backups) MeasureAll(ctx context.Context) error {
+// MeasureAll measures every item that is not a cache, one at a time, and keeps what it found. An item
+// measured by a walk is walked only when walks is true or its last walk is older than walkEvery: the
+// hourly round reads counters, and a walk runs at most once a day.
+func (b *Backups) MeasureAll(ctx context.Context, walks bool) error {
 	measuring.Lock()
 	defer measuring.Unlock()
 	items, err := b.Items()
 	if err != nil {
 		return err
 	}
+	before := b.Measurements()
 	found := map[string]map[string]Measured{}
 	for module, its := range items {
 		for _, it := range its {
@@ -293,6 +380,11 @@ func (b *Backups) MeasureAll(ctx context.Context) error {
 			if found[module] == nil {
 				found[module] = map[string]Measured{}
 			}
+			if was, ok := before[module][it.ID]; ok && it.Measure == measureWalk && !walks && was.MeasuredAt != nil &&
+				b.Now().Sub(*was.MeasuredAt) < walkEvery {
+				found[module][it.ID] = was
+				continue
+			}
 			found[module][it.ID] = b.measure(ctx, it)
 		}
 	}

modules/restic/cmd/restic-backups/data_test.go

@@ -6,6 +6,7 @@ package main
 import (
 	"context"
 	"errors"
+	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
@@ -57,12 +58,12 @@ func TestEveryItemButACacheIsMeasuredAndSaidWithItsLastGoodBackup(t *testing.T)
 		switch {
 		case name == "test":
 			return "", nil
-		case name == "sh" && strings.Contains(args[1], "find '"):
+		case name == "bash" && strings.Contains(args[1], "find '"):
 			walked = append(walked, args[1])
 			if strings.Contains(args[1], "'/var/lib/mesh-store'") {
-				return "1073741824 1759744800\n", nil
+				return "1073741824 1759744800 4000\n0\n", nil
 			}
-			return "5000 1759700000\n", nil
+			return "5000 1759700000 3\n0\n", nil
 		case name == "restic":
 			return "[]", nil
 		}
@@ -70,7 +71,7 @@ func TestEveryItemButACacheIsMeasuredAndSaidWithItsLastGoodBackup(t *testing.T)
 	}
 	night := time.Date(2026, 10, 6, 3, 0, 0, 0, time.UTC)
 	b := &Backups{Where: where, Run: run, Now: func() time.Time { return night.Add(time.Hour) }, Say: quiet}
-	must(t, b.MeasureAll(context.Background()))
+	must(t, b.MeasureAll(context.Background(), true))
 	if len(walked) != 2 {
 		t.Fatalf("walked %d paths, want the two that are not a cache: %v", len(walked), walked)
 	}
@@ -239,3 +240,66 @@ func TestTheRedundantStorageUnderAnItemIsRead(t *testing.T) {
 		t.Fatalf("plain storage read as redundant: %+v", r)
 	}
 }
+
+// Large items are never walked: a dataset is measured from the filesystem's counters and its top-level
+// entries, a shallow item from its top-level entries only; a walk that meets more than its bound stops
+// and says so; and a walk runs at most once a day — the hourly round keeps the last one.
+func TestNothingLargeIsWalkedAndAWalkIsBoundedAndDaily(t *testing.T) {
+	data := "# media\nitem movies irreplaceable /storage/media/movies - redundancy dataset\n" +
+		"item meta rebuildable /mnt/plex/config /mnt/plex/config backup shallow\n" +
+		"item big valuable /srv/big /srv/big backup\n"
+	where := withData(t, composed, data)
+	var mu sync.Mutex
+	var walks []string
+	truncated := true
+	run := func(_ context.Context, name string, args ...string) (string, error) {
+		mu.Lock()
+		defer mu.Unlock()
+		line := name + " " + strings.Join(args, " ")
+		switch {
+		case name == "test":
+			return "", nil
+		case name == "zfs" && args[len(args)-1] == "/storage/media/movies":
+			return "storage/media\t97067690722560\n", nil
+		case name == "zfs":
+			return "", errors.New("not a ZFS dataset")
+		case name == "bash" && strings.Contains(line, "-maxdepth 1"):
+			return "1759744800 12673\n", nil
+		case name == "bash":
+			walks = append(walks, line)
+			if strings.Contains(line, "-xdev") && !strings.Contains(line, "'/srv/big'") {
+				t.Errorf("walked something declared not to be walked: %s", line)
+			}
+			if truncated {
+				return fmt.Sprintf("123 1759700000 %d\n141\n", walkFiles), nil
+			}
+			return "999 1759700000 10\n0\n", nil
+		}
+		return "", nil
+	}
+	now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
+	b := &Backups{Where: where, Run: run, Now: func() time.Time { return now }, Say: quiet}
+	must(t, b.MeasureAll(context.Background(), false))
+	got := b.Measurements()["media"]
+	if m := got["movies"]; m.SizeBytes == nil || *m.SizeBytes != 97067690722560 || !strings.HasPrefix(m.Precision, "dataset storage/media") || m.LastWrite == nil {
+		t.Fatalf("the library from its dataset: %+v", m)
+	}
+	if m := got["meta"]; m.SizeBytes != nil || !strings.HasPrefix(m.Precision, "no size") || m.LastWrite == nil {
+		t.Fatalf("a shallow item: %+v", m)
+	}
+	if m := got["big"]; !strings.HasPrefix(m.Precision, "partial") {
+		t.Fatalf("a walk stopped at its bound: %+v", m)
+	}
+	// An hour later the hourly round reads counters again and walks nothing.
+	now = now.Add(time.Hour)
+	truncated = false
+	must(t, b.MeasureAll(context.Background(), false))
+	if len(walks) != 1 {
+		t.Fatalf("walked %d times in two rounds an hour apart: %v", len(walks), walks)
+	}
+	// After the night, it walks.
+	must(t, b.MeasureAll(context.Background(), true))
+	if len(walks) != 2 || b.Measurements()["media"]["big"].Precision != "exact" {
+		t.Fatalf("%d walks, %+v", len(walks), b.Measurements()["media"]["big"])
+	}
+}

modules/restic/cmd/restic-backups/main.go

@@ -58,7 +58,8 @@ func nights(b *Backups) {
 }
 
 // measurements measures every declared item once an hour (MESH_BACKUP_MEASURE_EVERY to change it),
-// the first a few minutes after start, and after every night (novox/hq ADR 0233).
+// the first a few minutes after start — counters hourly, a walk at most daily — and every walk after
+// each night (novox/hq ADR 0233).
 func measurements(b *Backups) {
 	every := time.Hour
 	if d, err := time.ParseDuration(os.Getenv("MESH_BACKUP_MEASURE_EVERY")); err == nil && d >= time.Minute {
@@ -66,7 +67,7 @@ func measurements(b *Backups) {
 	}
 	time.Sleep(3 * time.Minute)
 	for {
-		if err := b.MeasureAll(context.Background()); err != nil {
+		if err := b.MeasureAll(context.Background(), false); err != nil {
 			say("measuring the data declared here failed: %v", err)
 		}
 		time.Sleep(every)
@@ -89,7 +90,7 @@ func night(b *Backups) {
 	if len(failed) > 0 {
 		say("the night left %s without a backup", strings.Join(failed, ", "))
 	}
-	if err := b.MeasureAll(ctx); err != nil {
+	if err := b.MeasureAll(ctx, true); err != nil {
 		say("measuring the data declared here failed: %v", err)
 	}
 	// Sundays, the repository's own integrity with a sample of the data read back.

modules/supabase/module.json

@@ -64,7 +64,17 @@
         "id": "db",
         "path": "${dir:db-data}",
         "class": "valuable",
-        "why": "every table; copied as live files, which may not restore \u2014 a dump is wanted"
+        "backup": {
+          "dump": "docker exec supabase-db pg_dumpall -h 127.0.0.1 -U supabase_admin > ${dir:dumps}/all.sql.partial && mv ${dir:dumps}/all.sql.partial ${dir:dumps}/all.sql",
+          "into": "dumps"
+        },
+        "why": "every table; copied by pg_dumpall inside the database's container (local connections are trusted there), since a running store's files are not a consistent copy"
+      },
+      {
+        "id": "dumps",
+        "path": "${dir:dumps}",
+        "class": "rebuildable",
+        "why": "last night's dump of the database, made again every night"
       },
       {
         "id": "storage",
@@ -111,6 +121,12 @@
       "mode": "0700",
       "owner": "105:106"
     },
+    {
+      "id": "dumps",
+      "type": "directory",
+      "path": "${dir:state}/dumps",
+      "mode": "0700"
+    },
     {
       "id": "db-config",
       "type": "directory",