Measure each item its own way; never compare a partial size (hq ADR 0233)

A walk over a large library every hour loads the array that protects it. An item now says how it
is measured — a bounded daily walk, a dataset's counters, or its top level only — and a size that
is a lower bound is kept as such and never read as a shrink.

internal/catalogue/data.go

@@ -102,6 +102,10 @@ type DataItem struct {
 	Redundancy string `json:"redundancy,omitempty"`
 	// Within is how old its last good backup may be; unsaid, DefaultBackupWithin.
 	Within string `json:"within,omitempty"`
+	// Measure is how the backup holder measures it: `walk` (the default — every file, at most daily and
+	// bounded, for small items), `dataset` (a ZFS dataset's own counters, hourly, nothing walked) or
+	// `shallow` (its top-level entries only, no size). A large item never says walk.
+	Measure string `json:"measure,omitempty"`
 	// Active is how long it may go unwritten before that is a fault — for data something is
 	// expected to write all the time. Unsaid, a quiet item is not a fault.
 	Active string `json:"active,omitempty"`
@@ -207,6 +211,21 @@ func (it DataItem) Protection() string {
 	return strings.Join(by, "+")
 }
 
+// The ways an item is measured.
+const (
+	MeasureWalk    = "walk"
+	MeasureDataset = "dataset"
+	MeasureShallow = "shallow"
+)
+
+// MeasuredBy is how the item is measured, with the default applied.
+func (it DataItem) MeasuredBy() string {
+	if it.Measure == "" {
+		return MeasureWalk
+	}
+	return it.Measure
+}
+
 // OwnedByModule is whether the item is in one of the module's own directories — the mesh's to retire —
 // rather than an operator's path it was given, which the mesh never retires and never deletes.
 func (it DataItem) OwnedByModule() bool { return strings.HasPrefix(it.Path, "${dir:") }
@@ -365,6 +384,11 @@ func (m Manifest) dataProblems() []string {
 		if it.Class == ClassCache && it.Redundancy != "" {
 			say("%s is a cache and says it is protected by redundancy; a cache is not protected", label)
 		}
+		switch it.Measure {
+		case "", MeasureWalk, MeasureDataset, MeasureShallow:
+		default:
+			say("%s's measure %q is walk, dataset or shallow", label, it.Measure)
+		}
 		if _, err := ParseDataDuration(it.Within); err != nil {
 			say("%s's within: %v", label, err)
 		}
@@ -541,7 +565,8 @@ func (m Manifest) derivedContributions() []SeatContribution {
 		case it.BackedUp():
 			covered = it.Path
 		}
-		described = append(described, fmt.Sprintf("item %s %s %s %s %s", it.ID, it.Class, it.Path, covered, it.Protection()))
+		described = append(described, fmt.Sprintf("item %s %s %s %s %s %s", it.ID, it.Class, it.Path, covered,
+			it.Protection(), it.MeasuredBy()))
 	}
 	var out []SeatContribution
 	if len(lines) > 0 {

internal/catalogue/data_test.go

@@ -55,6 +55,7 @@ func TestTheDataSectionIsRefusedWhereItIsWrong(t *testing.T) {
 		{"irreplaceable and protected by nothing", `{"own":[{"id":"a","path":"${dir:d}","class":"irreplaceable","backup":"none"}]}`, "protected one way or the other"},
 		{"a cache on redundancy", `{"own":[{"id":"a","path":"${dir:d}","class":"cache","redundancy":"an array"}]}`, "a cache is not protected"},
 		{"kept with something not required", `{"kept-by":{"s3-bucket":{"class":"irreplaceable"}}}`, "does not require"},
+		{"a measure it does not know", `{"own":[{"id":"a","path":"${dir:d}","class":"cache","measure":"du"}]}`, "walk, dataset or shallow"},
 		{"a cache expecting writes", `{"own":[{"id":"a","path":"${dir:d}","class":"cache","active":"1d"}]}`, "only irreplaceable and valuable"},
 		{"a machine path", `{"own":[{"id":"a","path":"/srv/a","class":"cache"}]}`, "names no machine path"},
 		{"a directory it does not declare", `{"own":[{"id":"a","path":"${dir:nowhere}","class":"cache"}]}`, "declares no directory"},
@@ -168,7 +169,7 @@ func TestTheBackupHoldersLinesAreDerivedFromTheData(t *testing.T) {
 		"resources":[{"id":"home","type":"directory","path":"${machine:account-home}/.agent","mode":"0700"}]}`)
 	media := mustParse(t, `{"module":"media","version":"1",
 		"accesses":[{"id":"films","mode":"read"}],
-		"data":{"own":[{"id":"films","path":"${access:films}","class":"irreplaceable","redundancy":"on a redundant array; no room for a copy"},
+		"data":{"own":[{"id":"films","path":"${access:films}","class":"irreplaceable","redundancy":"on a redundant array; no room for a copy","measure":"dataset"},
 			{"id":"meta","path":"${dir:meta}","class":"rebuildable"}]},
 		"resources":[{"id":"meta","type":"directory","mode":"0700"}]}`)
 	facts := map[string]string{"account-home": "/home/op"}
@@ -185,9 +186,9 @@ func TestTheBackupHoldersLinesAreDerivedFromTheData(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	want = "# agent\nitem home valuable /home/op/.agent /home/op/.agent backup\nitem tmp cache /home/op/.agent/tmp - none\n" +
-		"# media\nitem films irreplaceable /tank/films - redundancy\nitem meta rebuildable /var/lib/media/meta /var/lib/media/meta backup\n" +
-		"# pg\nitem store irreplaceable /srv/store /srv/store/dumps backup\nitem dumps rebuildable /srv/store/dumps - none\n"
+	want = "# agent\nitem home valuable /home/op/.agent /home/op/.agent backup walk\nitem tmp cache /home/op/.agent/tmp - none walk\n" +
+		"# media\nitem films irreplaceable /tank/films - redundancy dataset\nitem meta rebuildable /var/lib/media/meta /var/lib/media/meta backup walk\n" +
+		"# pg\nitem store irreplaceable /srv/store /srv/store/dumps backup walk\nitem dumps rebuildable /srv/store/dumps - none walk\n"
 	if data != want {
 		t.Fatalf("data lines:\n%s\nwant:\n%s", data, want)
 	}