package health import ( "encoding/json" "fmt" "reflect" "strings" "time" "git.novox.be/novox/mesh-sdk/go/catalogue" ) // **Disk usage readings and peaks per data item** (novox/hq ADR 0571 clause 3, ADR 0233 §21, §27, §34): disk-load, on every // node, measures each data item the modules of its node declare (catalogue.PlacedDataItem, from compose/frombus.DataItems) // and keeps the newest reading and the item's largest size in seven days as its own state `readings`, one key // `..`, written only when it changes. The node-backup holder, mounts, nfs-server and the self-check's // probes read it there; the controller keeps none of it. Moved here from mesh-catalog modules/disk-load (b4c237d2). const ( DiskLoadModule = "disk-load" ReadingsState = "readings" ReadingsBucket = DiskLoadModule + "_" + ReadingsState ReadingsRead = DiskLoadModule + "." + ReadingsState ) // ReadingVersion is the version of a reading's layout. const ReadingVersion = 1 // PeakWindow is how far back the peak looks (ADR 0233 §34). const PeakWindow = 7 * 24 * time.Hour // Redundancy is the redundant storage an item is on, as read: zfs, md or btrfs, its pool, array or filesystem, and its // health (nil when it could not be read). type Redundancy struct { Kind string `json:"kind"` Where string `json:"where"` Healthy *bool `json:"healthy"` Said string `json:"said"` } // Measured is what one measurement found. Precision says what the size is: "exact", "dataset : its whole size", // "partial: ..." (a lower bound, never compared), or "no size: ...". type Measured struct { SizeBytes *int64 `json:"size_bytes,omitempty"` LastWrite *time.Time `json:"last_write,omitempty"` MeasuredAt *time.Time `json:"measured_at,omitempty"` Precision string `json:"precision,omitempty"` Error string `json:"error,omitempty"` Redundancy *Redundancy `json:"redundancy,omitempty"` } // Partial says the size is only a lower bound, which a peak is never made of. func (m Measured) Partial() bool { return strings.HasPrefix(m.Precision, "partial") } // DaySize is the largest size measured on one UTC day. type DaySize struct { Day string `json:"day"` Bytes int64 `json:"bytes"` } // Reading is the value of one key: the item as placed, what was measured, and the largest size in the seven days up to // it (Peak, measured on PeakAt), read from the day maxima Days. type Reading struct { V int `json:"v"` Node string `json:"node"` catalogue.PlacedDataItem Measured Peak *int64 `json:"peak_bytes,omitempty"` PeakAt string `json:"peak_day,omitempty"` Days []DaySize `json:"days,omitempty"` // FirstSeen is when the item was first measured on this node, kept by every reading after; RetiredAt and RetiredWhy // say its module left the node, written by whoever retires it and carried on unchanged. Each is left out while zero, // so a reading written before reads as it did. FirstSeen time.Time `json:"first_seen,omitzero"` RetiredAt time.Time `json:"retired_at,omitzero"` RetiredWhy string `json:"retired_why,omitempty"` } // ReadingKey is the key of an item on a node: `..`. func ReadingKey(node, module, item string) string { return node + "." + module + "." + item } // DecodeReading reads a key's value; it refuses another version. func DecodeReading(raw []byte) (Reading, error) { var r Reading if err := json.Unmarshal(raw, &r); err != nil { return Reading{}, fmt.Errorf("health: a reading is not readable: %w", err) } if r.V != ReadingVersion { return Reading{}, fmt.Errorf("health: a reading of version %d, and this reads %d", r.V, ReadingVersion) } return r, nil } // NextReading is the reading after a measurement, and whether it differs from the one before in anything but the time it // was taken: a reading that did not change is not written. A measurement with no size (an error, a shallow item) carries // the last size's days on, so the peak outlives one failed measurement. func NextReading(prev *Reading, node string, it catalogue.PlacedDataItem, m Measured, now time.Time) (Reading, bool) { r := Reading{V: ReadingVersion, Node: node, PlacedDataItem: it, Measured: m, FirstSeen: now.UTC()} var days []DaySize if prev != nil { days = prev.Days r.RetiredAt, r.RetiredWhy = prev.RetiredAt, prev.RetiredWhy if !prev.FirstSeen.IsZero() { r.FirstSeen = prev.FirstSeen } } today := now.UTC().Format("2006-01-02") cut := now.UTC().Add(-PeakWindow).Format("2006-01-02") var kept []DaySize todays := int64(-1) for _, d := range days { switch { case d.Day == today: todays = d.Bytes case d.Day >= cut: kept = append(kept, d) } } if m.SizeBytes != nil && !m.Partial() && *m.SizeBytes > todays { todays = *m.SizeBytes } if todays >= 0 { kept = append(kept, DaySize{Day: today, Bytes: todays}) } r.Days = kept for _, d := range kept { if r.Peak == nil || d.Bytes >= *r.Peak { b := d.Bytes r.Peak, r.PeakAt = &b, d.Day } } if prev == nil { return r, true } a, b := *prev, r a.MeasuredAt, b.MeasuredAt = nil, nil return r, !reflect.DeepEqual(a, b) } // **Each item's last good night** (novox/hq ADR 0571 clause 3, ADR 0564 §6; the lead, 2026-10-29): the node-backup holder // restic keeps, from its tools bundle through the runtime on the night's file event (its night's process runs as the // operator's account and holds no bus account, ADR 0436 §1-2), its own state `good`, one key `..` for every item a module // declares that a backup path covers, written when it changes: the item, the path whose snapshot keeps it and the newest // good night of its module (zero before the first). The self-check's probe of a stale backup reads it there. Moved here // from mesh-catalog modules/restic (e5ef3bed). const ( ResticModule = "restic" GoodState = "good" GoodBucket = ResticModule + "_" + GoodState GoodRead = ResticModule + "." + GoodState GoodVersion = 1 ) // GoodRow is one key of restic's `good`. type GoodRow struct { V int `json:"v"` Node string `json:"node"` Module string `json:"module"` Item string `json:"item"` CoveredBy string `json:"covered_by"` LastGood time.Time `json:"last_good,omitzero"` } // GoodKey is a row's key: `..`, as a reading's. func GoodKey(node, module, item string) string { return ReadingKey(node, module, item) } // DecodeGoodRow reads a row; it refuses another version. func DecodeGoodRow(raw []byte) (GoodRow, error) { var g GoodRow if err := json.Unmarshal(raw, &g); err != nil { return GoodRow{}, fmt.Errorf("health: a last good night is not readable: %w", err) } if g.V != GoodVersion { return GoodRow{}, fmt.Errorf("health: a last good night of version %d, and this reads %d", g.V, GoodVersion) } return g, nil }