novox/mesh-controller
6
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Measure each item its own way; never compare a partial size (hq ADR 0233)

Browse Source 
A walk over a large library every hour loads the array that protects it. An item now says how it
is measured — a bounded daily walk, a dataset's counters, or its top level only — and a size that
is a lower bound is kept as such and never read as a shrink.
This commit is contained in:
jochen
2026-10-06 17:00:22 +02:00
parent 52af210e47
commit abf9125689
7 changed files with 156 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/inventory/data.go
+30 -8
View File
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"strings"
"time"
"github.com/jackc/pgx/v5"
@@ -43,7 +44,9 @@ type Measurement struct {
MeasuredAt *time.Time
LastBackup *time.Time
// Error is what the holder could not measure, when it could not.
Error string
Error string
// Precision is what the size is, as the holder said it: "exact", "dataset …", "partial: …", "no size: …".
Precision string
Redundancy *Redundancy
}
@@ -54,6 +57,7 @@ type DataRecord struct {
Protection string
FirstSeen, DeclaredAt time.Time
MeasureError string
Precision string
Redundancy *Redundancy
Size *int64
LastWrite, MeasuredAt, LastBackup *time.Time
@@ -63,6 +67,21 @@ type DataRecord struct {
DeletedBy, DeletedWhy string
}
// Comparable is whether a size is fit to compare with another: exact, or a dataset's own counters.
func Comparable(precision string) bool {
return precision == "" || precision == "exact" || strings.HasPrefix(precision, "dataset ")
}
// Dataset is the ZFS dataset a size was read from, when it was: what several items on one dataset share.
func Dataset(precision string) string {
rest, ok := strings.CutPrefix(precision, "dataset ")
if !ok {
return ""
}
name, _, _ := strings.Cut(rest, ":")
return name
}
// Retired is whether the item is retired and not deleted.
func (r DataRecord) Retired() bool { return r.RetiredAt != nil && r.DeletedAt == nil }
@@ -139,12 +158,15 @@ func (i *Inventory) RecordData(ctx context.Context, machine string, declared []D
if _, err := tx.Exec(ctx, `
insert into data_item (machine, module, item, class, path, first_seen, declared_at,
size_bytes, last_write, measured_at, last_backup, owned, protection,
measure_error, redundancy_kind, redundancy_where, redundancy_healthy, redundancy_said)
values ($1, $2, $3, $4, $5, $6, $6, $7, $8, $9, $10, $12, $13, $14, $15, $16, $17, $18)
measure_error, redundancy_kind, redundancy_where, redundancy_healthy, redundancy_said,
precision)
values ($1, $2, $3, $4, $5, $6, $6, $7, $8, $9, $10, $12, $13, $14, $15, $16, $17, $18, $19)
on conflict (machine, module, item) do update set
class = excluded.class,
owned = excluded.owned,
protection = excluded.protection,
precision = case when excluded.measured_at is not null then excluded.precision else data_item.precision end,
size_bytes = case when excluded.measured_at is not null then excluded.size_bytes else data_item.size_bytes end,
measure_error = case when excluded.measured_at is not null then excluded.measure_error else data_item.measure_error end,
redundancy_kind = case when excluded.measured_at is not null then excluded.redundancy_kind else data_item.redundancy_kind end,
redundancy_where = case when excluded.measured_at is not null then excluded.redundancy_where else data_item.redundancy_where end,
@@ -153,17 +175,17 @@ func (i *Inventory) RecordData(ctx context.Context, machine string, declared []D
path = case when excluded.path <> '' then excluded.path else data_item.path end,
first_seen = case when $11::boolean then excluded.first_seen else data_item.first_seen end,
declared_at = excluded.declared_at,
size_bytes = coalesce(excluded.size_bytes, data_item.size_bytes),
last_write = coalesce(excluded.last_write, data_item.last_write),
measured_at = coalesce(excluded.measured_at, data_item.measured_at),
last_backup = coalesce(excluded.last_backup, data_item.last_backup),
retired_at = null, retired_why = null,
deleted_at = null, deleted_by = null, deleted_why = null`,
machine, d.Module, d.Item, d.Class, m.Path, now, m.Size, m.LastWrite, m.MeasuredAt, m.LastBackup,
fresh, d.Owned, d.Protection, nullable(m.Error), red.Kind, red.Where, red.Healthy, red.Said); err != nil {
fresh, d.Owned, d.Protection, nullable(m.Error), red.Kind, red.Where, red.Healthy, red.Said,
nullable(m.Precision)); err != nil {
return change, err
}
if m.Size != nil && m.MeasuredAt != nil {
if m.Size != nil && m.MeasuredAt != nil && Comparable(m.Precision) {
if _, err := tx.Exec(ctx, `
insert into data_reading (machine, module, item, at, size_bytes, last_write)
select $1, $2, $3, $4, $5, $6
@@ -203,7 +225,7 @@ func (i *Inventory) RecordData(ctx context.Context, machine string, declared []D
const dataSelect = `select machine, module, item, class, path, first_seen, declared_at, size_bytes, last_write,
measured_at, last_backup, retired_at, coalesce(retired_why, ''), deleted_at, coalesce(deleted_by, ''),
coalesce(deleted_why, ''), owned, protection, coalesce(measure_error, ''), redundancy_kind, redundancy_where,
redundancy_healthy, redundancy_said from data_item`
redundancy_healthy, redundancy_said, coalesce(precision, '') from data_item`
func scanData(rows pgx.Row) (DataRecord, error) {
var r DataRecord
@@ -211,7 +233,7 @@ func scanData(rows pgx.Row) (DataRecord, error) {
var healthy *bool
err := rows.Scan(&r.Machine, &r.Module, &r.Item, &r.Class, &r.Path, &r.FirstSeen, &r.DeclaredAt, &r.Size,
&r.LastWrite, &r.MeasuredAt, &r.LastBackup, &r.RetiredAt, &r.RetiredWhy, &r.DeletedAt, &r.DeletedBy,
&r.DeletedWhy, &r.Owned, &r.Protection, &r.MeasureError, &kind, &where, &healthy, &said)
&r.DeletedWhy, &r.Owned, &r.Protection, &r.MeasureError, &kind, &where, &healthy, &said, &r.Precision)
if err == nil && kind != nil {
r.Redundancy = &Redundancy{Kind: *kind, Healthy: healthy}
if where != nil {
internal/inventory/data_test.go
+21
View File
@@ -107,3 +107,24 @@ func TestReadingsAreHourlyAndThePeakIsTheLargest(t *testing.T) {
t.Fatalf("the newest measurement is not the one on record: %+v", r)
}
}
// A partial walk's lower bound is kept as the newest measurement, said as partial, and never kept as a
// reading a shrink would be read against.
func TestAPartialMeasurementIsNeverAReading(t *testing.T) {
inv := fresh(t)
ctx := t.Context()
now := time.Date(2026, 10, 6, 0, 0, 0, 0, time.UTC)
declared := []DeclaredData{{Module: "big", Item: "data", Class: "valuable", Owned: true}}
if _, err := inv.RecordData(ctx, "home", declared, map[string]map[string]Measurement{"big": {"data": {
Path: "/srv/big", Size: size(5), MeasuredAt: at(now), Precision: "partial: stopped"}}}, "", now); err != nil {
t.Fatal(err)
}
var n int
if err := inv.store.Pool().QueryRow(ctx, `select count(*) from data_reading`).Scan(&n); err != nil || n != 0 {
t.Fatalf("%d readings from a partial measurement: %v", n, err)
}
r, err := inv.DataOf(ctx, "home", "big", "data")
if err != nil || r.Precision != "partial: stopped" || Comparable(r.Precision) {
t.Fatalf("%+v, %v", r, err)
}
}
internal/inventory/migrations/0072-a-module-declares-the-data-it-holds.sql
+4 -1
View File
@@ -38,6 +38,8 @@ create table data_item (
last_backup timestamptz,
-- What the holder could not measure, when it could not: the path gone, a walk refused.
measure_error text,
-- What the newest size is: exact, a dataset's whole size, partial (a lower bound) or none.
precision text,
-- The redundant storage it is on, as the holder read it: zfs, md or btrfs, which pool or device,
-- whether it is healthy, and what it said.
redundancy_kind text,
@@ -53,7 +55,8 @@ create table data_item (
);
-- One row per measurement kept, at most one an hour per item, for ninety days: what a shrink is
-- read against.
-- read against. Only a comparable size is kept: exact, or a dataset's own counters — never a partial
-- walk's lower bound.
create table data_reading (
machine text not null,
module text not null,