novox/mesh-controller
6
0
Fork 0
Code Issues Pull Requests 3 Releases Wiki Activity

Measure each item its own way; never compare a partial size (hq ADR 0233)

Browse Source 
A walk over a large library every hour loads the array that protects it. An item now says how it
is measured — a bounded daily walk, a dataset's counters, or its top level only — and a size that
is a lower bound is kept as such and never read as a shrink.
This commit is contained in:
jochen
2026-10-06 17:00:22 +02:00
parent 52af210e47
commit abf9125689
7 changed files with 156 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/inventory/data.go
+30 -8
View File
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"strings"
"time"
"github.com/jackc/pgx/v5"
@@ -43,7 +44,9 @@ type Measurement struct {
MeasuredAt *time.Time
LastBackup *time.Time
// Error is what the holder could not measure, when it could not.
Error string
Error string
// Precision is what the size is, as the holder said it: "exact", "dataset …", "partial: …", "no size: …".
Precision string
Redundancy *Redundancy
}
@@ -54,6 +57,7 @@ type DataRecord struct {
Protection string
FirstSeen, DeclaredAt time.Time
MeasureError string
Precision string
Redundancy *Redundancy
Size *int64
LastWrite, MeasuredAt, LastBackup *time.Time
@@ -63,6 +67,21 @@ type DataRecord struct {
DeletedBy, DeletedWhy string
}
// Comparable is whether a size is fit to compare with another: exact, or a dataset's own counters.
func Comparable(precision string) bool {
return precision == "" || precision == "exact" || strings.HasPrefix(precision, "dataset ")
}
// Dataset is the ZFS dataset a size was read from, when it was: what several items on one dataset share.
func Dataset(precision string) string {
rest, ok := strings.CutPrefix(precision, "dataset ")
if !ok {
return ""
}
name, _, _ := strings.Cut(rest, ":")
return name
}
// Retired is whether the item is retired and not deleted.
func (r DataRecord) Retired() bool { return r.RetiredAt != nil && r.DeletedAt == nil }
@@ -139,12 +158,15 @@ func (i *Inventory) RecordData(ctx context.Context, machine string, declared []D
if _, err := tx.Exec(ctx, `
insert into data_item (machine, module, item, class, path, first_seen, declared_at,
size_bytes, last_write, measured_at, last_backup, owned, protection,
measure_error, redundancy_kind, redundancy_where, redundancy_healthy, redundancy_said)
values ($1, $2, $3, $4, $5, $6, $6, $7, $8, $9, $10, $12, $13, $14, $15, $16, $17, $18)
measure_error, redundancy_kind, redundancy_where, redundancy_healthy, redundancy_said,
precision)
values ($1, $2, $3, $4, $5, $6, $6, $7, $8, $9, $10, $12, $13, $14, $15, $16, $17, $18, $19)
on conflict (machine, module, item) do update set
class = excluded.class,
owned = excluded.owned,
protection = excluded.protection,
precision = case when excluded.measured_at is not null then excluded.precision else data_item.precision end,
size_bytes = case when excluded.measured_at is not null then excluded.size_bytes else data_item.size_bytes end,
measure_error = case when excluded.measured_at is not null then excluded.measure_error else data_item.measure_error end,
redundancy_kind = case when excluded.measured_at is not null then excluded.redundancy_kind else data_item.redundancy_kind end,
redundancy_where = case when excluded.measured_at is not null then excluded.redundancy_where else data_item.redundancy_where end,
@@ -153,17 +175,17 @@ func (i *Inventory) RecordData(ctx context.Context, machine string, declared []D
path = case when excluded.path <> '' then excluded.path else data_item.path end,
first_seen = case when $11::boolean then excluded.first_seen else data_item.first_seen end,
declared_at = excluded.declared_at,
size_bytes = coalesce(excluded.size_bytes, data_item.size_bytes),
last_write = coalesce(excluded.last_write, data_item.last_write),
measured_at = coalesce(excluded.measured_at, data_item.measured_at),
last_backup = coalesce(excluded.last_backup, data_item.last_backup),
retired_at = null, retired_why = null,
deleted_at = null, deleted_by = null, deleted_why = null`,
machine, d.Module, d.Item, d.Class, m.Path, now, m.Size, m.LastWrite, m.MeasuredAt, m.LastBackup,
fresh, d.Owned, d.Protection, nullable(m.Error), red.Kind, red.Where, red.Healthy, red.Said); err != nil {
fresh, d.Owned, d.Protection, nullable(m.Error), red.Kind, red.Where, red.Healthy, red.Said,
nullable(m.Precision)); err != nil {
return change, err
}
if m.Size != nil && m.MeasuredAt != nil {
if m.Size != nil && m.MeasuredAt != nil && Comparable(m.Precision) {
if _, err := tx.Exec(ctx, `
insert into data_reading (machine, module, item, at, size_bytes, last_write)
select $1, $2, $3, $4, $5, $6
@@ -203,7 +225,7 @@ func (i *Inventory) RecordData(ctx context.Context, machine string, declared []D
const dataSelect = `select machine, module, item, class, path, first_seen, declared_at, size_bytes, last_write,
measured_at, last_backup, retired_at, coalesce(retired_why, ''), deleted_at, coalesce(deleted_by, ''),
coalesce(deleted_why, ''), owned, protection, coalesce(measure_error, ''), redundancy_kind, redundancy_where,
redundancy_healthy, redundancy_said from data_item`
redundancy_healthy, redundancy_said, coalesce(precision, '') from data_item`
func scanData(rows pgx.Row) (DataRecord, error) {
var r DataRecord
@@ -211,7 +233,7 @@ func scanData(rows pgx.Row) (DataRecord, error) {
var healthy *bool
err := rows.Scan(&r.Machine, &r.Module, &r.Item, &r.Class, &r.Path, &r.FirstSeen, &r.DeclaredAt, &r.Size,
&r.LastWrite, &r.MeasuredAt, &r.LastBackup, &r.RetiredAt, &r.RetiredWhy, &r.DeletedAt, &r.DeletedBy,
&r.DeletedWhy, &r.Owned, &r.Protection, &r.MeasureError, &kind, &where, &healthy, &said)
&r.DeletedWhy, &r.Owned, &r.Protection, &r.MeasureError, &kind, &where, &healthy, &said, &r.Precision)
if err == nil && kind != nil {
r.Redundancy = &Redundancy{Kind: *kind, Healthy: healthy}
if where != nil {