novox/mesh-controller
6
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Measure each item its own way; never compare a partial size (hq ADR 0233)

Browse Source 
A walk over a large library every hour loads the array that protects it. An item now says how it
is measured — a bounded daily walk, a dataset's counters, or its top level only — and a size that
is a lower bound is kept as such and never read as a shrink.
This commit is contained in:
jochen
2026-10-06 17:00:22 +02:00
parent 52af210e47
commit abf9125689
7 changed files with 156 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-controller/data.go
+40 -5
View File
@@ -95,6 +95,8 @@ type holderItem struct {
MeasuredAt *time.Time `json:"measured_at"`
LastBackup *time.Time `json:"last_backup"`
Error string `json:"error,omitempty"`
// Precision is what the size is: exact, a dataset's, partial, or none (ADR 0233).
Precision string `json:"precision,omitempty"`
// Redundancy is the redundant storage the item is on, where the holder could tell (ADR 0233).
Redundancy *inventory.Redundancy `json:"redundancy,omitempty"`
}
@@ -112,7 +114,8 @@ func readHolder(raw json.RawMessage) (map[string]map[string]inventory.Measuremen
out[m.Module] = map[string]inventory.Measurement{}
}
out[m.Module][it.Item] = inventory.Measurement{Path: it.Path, Size: it.SizeBytes, LastWrite: it.LastWrite,
MeasuredAt: it.MeasuredAt, LastBackup: it.LastBackup, Error: it.Error, Redundancy: it.Redundancy}
MeasuredAt: it.MeasuredAt, LastBackup: it.LastBackup, Error: it.Error, Redundancy: it.Redundancy,
Precision: it.Precision}
}
}
return out, nil
@@ -377,6 +380,12 @@ func dataFindings(records []inventory.DataRecord, peaks map[string]int64, shelf
var out []conditions.Observation
byItem := map[string][]inventory.DataRecord{}
arrays := map[string][]inventory.DataRecord{}
type shrunk struct {
machine, dataset, class string
size, peak int64
items []string
}
shrunkDatasets := map[string]shrunk{}
for _, r := range records {
if r.DeletedAt != nil {
continue
@@ -416,7 +425,18 @@ func dataFindings(records []inventory.DataRecord, peaks map[string]int64, shelf
Kind: kindDataMissing, Machine: r.Machine, Severity: severity, Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("%s of %s on %s (%s) is gone: %s does not exist any more", r.Item, r.Module, r.Machine,
class, orUnknownPath(r.Path))})
} else if peak, ok := peaks[r.Key()]; ok && r.Size != nil && *r.Size*2 < peak && peak-*r.Size >= shrinkFloor {
} else if peak, ok := peaks[r.Key()]; ok && r.Size != nil && inventory.Comparable(r.Precision) &&
*r.Size*2 < peak && peak-*r.Size >= shrinkFloor && inventory.Dataset(r.Precision) != "" {
// Several items on one dataset share its size: one condition for the dataset, as loud as the
// most precious item on it.
k := r.Machine + "/" + inventory.Dataset(r.Precision)
ds := shrunkDatasets[k]
ds.machine, ds.dataset, ds.size, ds.peak = r.Machine, inventory.Dataset(r.Precision), *r.Size, peak
ds.class = catalogue.StricterClass(ds.class, class)
ds.items = append(ds.items, r.Module+"/"+r.Item)
shrunkDatasets[k] = ds
} else if peak, ok := peaks[r.Key()]; ok && r.Size != nil && inventory.Comparable(r.Precision) &&
*r.Size*2 < peak && peak-*r.Size >= shrinkFloor {
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: id, Token: kindDataShrank,
Kind: kindDataShrank, Machine: r.Machine, Severity: severity, Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("%s of %s on %s (%s) shrank to %s from %s within %d days — more than half of what it "+
@@ -453,6 +473,15 @@ func dataFindings(records []inventory.DataRecord, peaks map[string]int64, shelf
}
}
}
for _, k := range keysSorted(shrunkDatasets) {
ds := shrunkDatasets[k]
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine,
ID: ds.machine + ".dataset." + strings.ReplaceAll(ds.dataset, "/", "-"), Token: kindDataShrank,
Kind: kindDataShrank, Machine: ds.machine, Severity: severityOf(ds.class), Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("the dataset %s on %s shrank to %s from %s within %d days — more than half of what it held "+
"is gone; it holds %s", ds.dataset, ds.machine, sizeWords(&ds.size), sizeWords(&ds.peak),
int(shrinkWindow.Hours()/24), strings.Join(ds.items, ", "))})
}
// The redundant storage watched data is on: one condition per array, as loud as the most precious
// item on it — the array, not each item, is what degrades.
for _, where := range keysSorted(arrays) {
@@ -484,11 +513,12 @@ func dataFindings(records []inventory.DataRecord, peaks map[string]int64, shelf
for _, key := range keysSorted(byItem) {
rs := byItem[key]
for _, a := range rs {
if a.Retired() || a.Size == nil || !catalogue.Watched(a.Class) {
if a.Retired() || a.Size == nil || !catalogue.Watched(a.Class) || !inventory.Comparable(a.Precision) {
continue
}
for _, o := range rs {
if o.Machine == a.Machine || !o.Retired() || o.Size == nil || !replacedByLess(*a.Size, *o.Size) {
if o.Machine == a.Machine || !o.Retired() || o.Size == nil || !inventory.Comparable(o.Precision) ||
!replacedByLess(*a.Size, *o.Size) {
continue
}
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine,
@@ -603,6 +633,8 @@ type dataRow struct {
// Array is the redundant storage it is on and its state, where its holder could tell.
Array string `json:"array,omitempty"`
Unmeasured string `json:"unmeasured,omitempty"`
// Precision says what the size is: exact, a dataset's whole size, partial, or none.
Precision string `json:"precision,omitempty"`
SizeBytes *int64 `json:"size-bytes,omitempty"`
LastWrite string `json:"last-write,omitempty"`
MeasuredAt string `json:"measured-at,omitempty"`
@@ -659,6 +691,9 @@ func dataCommand(ctx context.Context, args []string) error {
if r.Array != "" {
fmt.Printf(" on %s\n", r.Array)
}
if r.Precision != "" && r.Precision != "exact" {
fmt.Printf(" size: %s\n", r.Precision)
}
if r.Unmeasured != "" {
fmt.Printf(" not measured: %s\n", r.Unmeasured)
}
@@ -687,7 +722,7 @@ func dataRows(records []inventory.DataRecord, shelf map[string]catalogue.Manifes
continue
}
row := dataRow{Machine: r.Machine, Module: r.Module, Item: r.Item, Class: r.Class, Path: r.Path,
Protection: r.Protection, Unmeasured: r.MeasureError, SizeBytes: r.Size, LastWrite: stamp(r.LastWrite), MeasuredAt: stamp(r.MeasuredAt),
Protection: r.Protection, Unmeasured: r.MeasureError, Precision: r.Precision, SizeBytes: r.Size, LastWrite: stamp(r.LastWrite), MeasuredAt: stamp(r.MeasuredAt),
LastBackup: stamp(r.LastBackup), Retired: stamp(r.RetiredAt), RetiredWhy: r.RetiredWhy,
Deleted: stamp(r.DeletedAt)}
if it, ok := shelf[r.Module].DataItem(r.Item); ok && it.BackedUp() {
cmd/mesh-controller/data_test.go
+30
View File
@@ -428,3 +428,33 @@ func TestKeptByHoldsTheProvidersItemToTheConsumersClass(t *testing.T) {
t.Fatalf("a valuable store without a backup: %+v", o)
}
}
// Items measured from one dataset's counters share its size: a shrink of the dataset is one condition
// naming every item on it, not one per item; and a partial walk's lower bound is never compared.
func TestADatasetShrinksOnceAndAPartialSizeIsNeverCompared(t *testing.T) {
now := time.Now()
media := `{"module":"media","version":"1","accesses":[{"id":"films","mode":"read"},{"id":"shows","mode":"read"}],
"data":{"own":[{"id":"films","path":"${access:films}","class":"irreplaceable","redundancy":"an array","measure":"dataset"},
{"id":"shows","path":"${access:shows}","class":"irreplaceable","redundancy":"an array","measure":"dataset"}]}}`
shelf := shelfFor(t, media, houseManifest)
well := true
on := func(item string, size int64) inventory.DataRecord {
return inventory.DataRecord{Machine: "home", Module: "media", Item: item, Class: "irreplaceable",
Size: bytesOf(size), FirstSeen: now.Add(-24 * time.Hour), MeasuredAt: when(now),
Precision: "dataset tank/media: its whole size",
Redundancy: &inventory.Redundancy{Kind: "zfs", Where: "tank", Healthy: &well}}
}
films, shows := on("films", 30<<40), on("shows", 30<<40)
peaks := map[string]int64{films.Key(): 90 << 40, shows.Key(): 90 << 40}
got := dataFindings([]inventory.DataRecord{films, shows}, peaks, shelf, nil, nil, now)
if len(got) != 1 || got[0].Kind != kindDataShrank || got[0].Severity != conditions.Urgent ||
!strings.Contains(got[0].Summary, "media/films, media/shows") {
t.Fatalf("%+v", got)
}
partial := inventory.DataRecord{Machine: "home", Module: "house", Item: "config", Class: "irreplaceable",
Size: bytesOf(1 << 20), FirstSeen: now.Add(-24 * time.Hour), MeasuredAt: when(now), LastWrite: when(now),
LastBackup: when(now), Precision: "partial: measured partially"}
if got := dataFindings([]inventory.DataRecord{partial}, map[string]int64{partial.Key(): 1 << 30}, shelf, nil, nil, now); len(got) != 0 {
t.Fatalf("a partial size was compared: %+v", got)
}
}
internal/catalogue/data.go
+26 -1
View File
@@ -102,6 +102,10 @@ type DataItem struct {
Redundancy string `json:"redundancy,omitempty"`
// Within is how old its last good backup may be; unsaid, DefaultBackupWithin.
Within string `json:"within,omitempty"`
// Measure is how the backup holder measures it: `walk` (the default — every file, at most daily and
// bounded, for small items), `dataset` (a ZFS dataset's own counters, hourly, nothing walked) or
// `shallow` (its top-level entries only, no size). A large item never says walk.
Measure string `json:"measure,omitempty"`
// Active is how long it may go unwritten before that is a fault — for data something is
// expected to write all the time. Unsaid, a quiet item is not a fault.
Active string `json:"active,omitempty"`
@@ -207,6 +211,21 @@ func (it DataItem) Protection() string {
return strings.Join(by, "+")
}
// The ways an item is measured.
const (
MeasureWalk = "walk"
MeasureDataset = "dataset"
MeasureShallow = "shallow"
)
// MeasuredBy is how the item is measured, with the default applied.
func (it DataItem) MeasuredBy() string {
if it.Measure == "" {
return MeasureWalk
}
return it.Measure
}
// OwnedByModule is whether the item is in one of the module's own directories — the mesh's to retire —
// rather than an operator's path it was given, which the mesh never retires and never deletes.
func (it DataItem) OwnedByModule() bool { return strings.HasPrefix(it.Path, "${dir:") }
@@ -365,6 +384,11 @@ func (m Manifest) dataProblems() []string {
if it.Class == ClassCache && it.Redundancy != "" {
say("%s is a cache and says it is protected by redundancy; a cache is not protected", label)
}
switch it.Measure {
case "", MeasureWalk, MeasureDataset, MeasureShallow:
default:
say("%s's measure %q is walk, dataset or shallow", label, it.Measure)
}
if _, err := ParseDataDuration(it.Within); err != nil {
say("%s's within: %v", label, err)
}
@@ -541,7 +565,8 @@ func (m Manifest) derivedContributions() []SeatContribution {
case it.BackedUp():
covered = it.Path
}
described = append(described, fmt.Sprintf("item %s %s %s %s %s", it.ID, it.Class, it.Path, covered, it.Protection()))
described = append(described, fmt.Sprintf("item %s %s %s %s %s %s", it.ID, it.Class, it.Path, covered,
it.Protection(), it.MeasuredBy()))
}
var out []SeatContribution
if len(lines) > 0 {
internal/catalogue/data_test.go
+5 -4
View File
@@ -55,6 +55,7 @@ func TestTheDataSectionIsRefusedWhereItIsWrong(t *testing.T) {
{"irreplaceable and protected by nothing", `{"own":[{"id":"a","path":"${dir:d}","class":"irreplaceable","backup":"none"}]}`, "protected one way or the other"},
{"a cache on redundancy", `{"own":[{"id":"a","path":"${dir:d}","class":"cache","redundancy":"an array"}]}`, "a cache is not protected"},
{"kept with something not required", `{"kept-by":{"s3-bucket":{"class":"irreplaceable"}}}`, "does not require"},
{"a measure it does not know", `{"own":[{"id":"a","path":"${dir:d}","class":"cache","measure":"du"}]}`, "walk, dataset or shallow"},
{"a cache expecting writes", `{"own":[{"id":"a","path":"${dir:d}","class":"cache","active":"1d"}]}`, "only irreplaceable and valuable"},
{"a machine path", `{"own":[{"id":"a","path":"/srv/a","class":"cache"}]}`, "names no machine path"},
{"a directory it does not declare", `{"own":[{"id":"a","path":"${dir:nowhere}","class":"cache"}]}`, "declares no directory"},
@@ -168,7 +169,7 @@ func TestTheBackupHoldersLinesAreDerivedFromTheData(t *testing.T) {
"resources":[{"id":"home","type":"directory","path":"${machine:account-home}/.agent","mode":"0700"}]}`)
media := mustParse(t, `{"module":"media","version":"1",
"accesses":[{"id":"films","mode":"read"}],
"data":{"own":[{"id":"films","path":"${access:films}","class":"irreplaceable","redundancy":"on a redundant array; no room for a copy"},
"data":{"own":[{"id":"films","path":"${access:films}","class":"irreplaceable","redundancy":"on a redundant array; no room for a copy","measure":"dataset"},
{"id":"meta","path":"${dir:meta}","class":"rebuildable"}]},
"resources":[{"id":"meta","type":"directory","mode":"0700"}]}`)
facts := map[string]string{"account-home": "/home/op"}
@@ -185,9 +186,9 @@ func TestTheBackupHoldersLinesAreDerivedFromTheData(t *testing.T) {
if err != nil {
t.Fatal(err)
}
want = "# agent\nitem home valuable /home/op/.agent /home/op/.agent backup\nitem tmp cache /home/op/.agent/tmp - none\n" +
"# media\nitem films irreplaceable /tank/films - redundancy\nitem meta rebuildable /var/lib/media/meta /var/lib/media/meta backup\n" +
"# pg\nitem store irreplaceable /srv/store /srv/store/dumps backup\nitem dumps rebuildable /srv/store/dumps - none\n"
want = "# agent\nitem home valuable /home/op/.agent /home/op/.agent backup walk\nitem tmp cache /home/op/.agent/tmp - none walk\n" +
"# media\nitem films irreplaceable /tank/films - redundancy dataset\nitem meta rebuildable /var/lib/media/meta /var/lib/media/meta backup walk\n" +
"# pg\nitem store irreplaceable /srv/store /srv/store/dumps backup walk\nitem dumps rebuildable /srv/store/dumps - none walk\n"
if data != want {
t.Fatalf("data lines:\n%s\nwant:\n%s", data, want)
}
internal/inventory/data.go
+30 -8
View File
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"strings"
"time"
"github.com/jackc/pgx/v5"
@@ -43,7 +44,9 @@ type Measurement struct {
MeasuredAt *time.Time
LastBackup *time.Time
// Error is what the holder could not measure, when it could not.
Error string
Error string
// Precision is what the size is, as the holder said it: "exact", "dataset …", "partial: …", "no size: …".
Precision string
Redundancy *Redundancy
}
@@ -54,6 +57,7 @@ type DataRecord struct {
Protection string
FirstSeen, DeclaredAt time.Time
MeasureError string
Precision string
Redundancy *Redundancy
Size *int64
LastWrite, MeasuredAt, LastBackup *time.Time
@@ -63,6 +67,21 @@ type DataRecord struct {
DeletedBy, DeletedWhy string
}
// Comparable is whether a size is fit to compare with another: exact, or a dataset's own counters.
func Comparable(precision string) bool {
return precision == "" || precision == "exact" || strings.HasPrefix(precision, "dataset ")
}
// Dataset is the ZFS dataset a size was read from, when it was: what several items on one dataset share.
func Dataset(precision string) string {
rest, ok := strings.CutPrefix(precision, "dataset ")
if !ok {
return ""
}
name, _, _ := strings.Cut(rest, ":")
return name
}
// Retired is whether the item is retired and not deleted.
func (r DataRecord) Retired() bool { return r.RetiredAt != nil && r.DeletedAt == nil }
@@ -139,12 +158,15 @@ func (i *Inventory) RecordData(ctx context.Context, machine string, declared []D
if _, err := tx.Exec(ctx, `
insert into data_item (machine, module, item, class, path, first_seen, declared_at,
size_bytes, last_write, measured_at, last_backup, owned, protection,
measure_error, redundancy_kind, redundancy_where, redundancy_healthy, redundancy_said)
values ($1, $2, $3, $4, $5, $6, $6, $7, $8, $9, $10, $12, $13, $14, $15, $16, $17, $18)
measure_error, redundancy_kind, redundancy_where, redundancy_healthy, redundancy_said,
precision)
values ($1, $2, $3, $4, $5, $6, $6, $7, $8, $9, $10, $12, $13, $14, $15, $16, $17, $18, $19)
on conflict (machine, module, item) do update set
class = excluded.class,
owned = excluded.owned,
protection = excluded.protection,
precision = case when excluded.measured_at is not null then excluded.precision else data_item.precision end,
size_bytes = case when excluded.measured_at is not null then excluded.size_bytes else data_item.size_bytes end,
measure_error = case when excluded.measured_at is not null then excluded.measure_error else data_item.measure_error end,
redundancy_kind = case when excluded.measured_at is not null then excluded.redundancy_kind else data_item.redundancy_kind end,
redundancy_where = case when excluded.measured_at is not null then excluded.redundancy_where else data_item.redundancy_where end,
@@ -153,17 +175,17 @@ func (i *Inventory) RecordData(ctx context.Context, machine string, declared []D
path = case when excluded.path <> '' then excluded.path else data_item.path end,
first_seen = case when $11::boolean then excluded.first_seen else data_item.first_seen end,
declared_at = excluded.declared_at,
size_bytes = coalesce(excluded.size_bytes, data_item.size_bytes),
last_write = coalesce(excluded.last_write, data_item.last_write),
measured_at = coalesce(excluded.measured_at, data_item.measured_at),
last_backup = coalesce(excluded.last_backup, data_item.last_backup),
retired_at = null, retired_why = null,
deleted_at = null, deleted_by = null, deleted_why = null`,
machine, d.Module, d.Item, d.Class, m.Path, now, m.Size, m.LastWrite, m.MeasuredAt, m.LastBackup,
fresh, d.Owned, d.Protection, nullable(m.Error), red.Kind, red.Where, red.Healthy, red.Said); err != nil {
fresh, d.Owned, d.Protection, nullable(m.Error), red.Kind, red.Where, red.Healthy, red.Said,
nullable(m.Precision)); err != nil {
return change, err
}
if m.Size != nil && m.MeasuredAt != nil {
if m.Size != nil && m.MeasuredAt != nil && Comparable(m.Precision) {
if _, err := tx.Exec(ctx, `
insert into data_reading (machine, module, item, at, size_bytes, last_write)
select $1, $2, $3, $4, $5, $6
@@ -203,7 +225,7 @@ func (i *Inventory) RecordData(ctx context.Context, machine string, declared []D
const dataSelect = `select machine, module, item, class, path, first_seen, declared_at, size_bytes, last_write,
measured_at, last_backup, retired_at, coalesce(retired_why, ''), deleted_at, coalesce(deleted_by, ''),
coalesce(deleted_why, ''), owned, protection, coalesce(measure_error, ''), redundancy_kind, redundancy_where,
redundancy_healthy, redundancy_said from data_item`
redundancy_healthy, redundancy_said, coalesce(precision, '') from data_item`
func scanData(rows pgx.Row) (DataRecord, error) {
var r DataRecord
@@ -211,7 +233,7 @@ func scanData(rows pgx.Row) (DataRecord, error) {
var healthy *bool
err := rows.Scan(&r.Machine, &r.Module, &r.Item, &r.Class, &r.Path, &r.FirstSeen, &r.DeclaredAt, &r.Size,
&r.LastWrite, &r.MeasuredAt, &r.LastBackup, &r.RetiredAt, &r.RetiredWhy, &r.DeletedAt, &r.DeletedBy,
&r.DeletedWhy, &r.Owned, &r.Protection, &r.MeasureError, &kind, &where, &healthy, &said)
&r.DeletedWhy, &r.Owned, &r.Protection, &r.MeasureError, &kind, &where, &healthy, &said, &r.Precision)
if err == nil && kind != nil {
r.Redundancy = &Redundancy{Kind: *kind, Healthy: healthy}
if where != nil {
internal/inventory/data_test.go
+21
View File
@@ -107,3 +107,24 @@ func TestReadingsAreHourlyAndThePeakIsTheLargest(t *testing.T) {
t.Fatalf("the newest measurement is not the one on record: %+v", r)
}
}
// A partial walk's lower bound is kept as the newest measurement, said as partial, and never kept as a
// reading a shrink would be read against.
func TestAPartialMeasurementIsNeverAReading(t *testing.T) {
inv := fresh(t)
ctx := t.Context()
now := time.Date(2026, 10, 6, 0, 0, 0, 0, time.UTC)
declared := []DeclaredData{{Module: "big", Item: "data", Class: "valuable", Owned: true}}
if _, err := inv.RecordData(ctx, "home", declared, map[string]map[string]Measurement{"big": {"data": {
Path: "/srv/big", Size: size(5), MeasuredAt: at(now), Precision: "partial: stopped"}}}, "", now); err != nil {
t.Fatal(err)
}
var n int
if err := inv.store.Pool().QueryRow(ctx, `select count(*) from data_reading`).Scan(&n); err != nil || n != 0 {
t.Fatalf("%d readings from a partial measurement: %v", n, err)
}
r, err := inv.DataOf(ctx, "home", "big", "data")
if err != nil || r.Precision != "partial: stopped" || Comparable(r.Precision) {
t.Fatalf("%+v, %v", r, err)
}
}
internal/inventory/migrations/0072-a-module-declares-the-data-it-holds.sql
+4 -1
View File
@@ -38,6 +38,8 @@ create table data_item (
last_backup timestamptz,
-- What the holder could not measure, when it could not: the path gone, a walk refused.
measure_error text,
-- What the newest size is: exact, a dataset's whole size, partial (a lower bound) or none.
precision text,
-- The redundant storage it is on, as the holder read it: zfs, md or btrfs, which pool or device,
-- whether it is healthy, and what it said.
redundancy_kind text,
@@ -53,7 +55,8 @@ create table data_item (
);
-- One row per measurement kept, at most one an hour per item, for ninety days: what a shrink is
-- read against.
-- read against. Only a comparable size is kept: exact, or a dataset's own counters — never a partial
-- walk's lower bound.
create table data_reading (
machine text not null,
module text not null,