health: readings.go (disk-load's readings, restic's last good night) leaves, held by disk-load's diskreadings and restic's backupgood (S-7 part, B9f; issue 1966)
Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i
This commit is contained in:
@@ -7,8 +7,6 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// ADR 0564 clause 3 (issue 1952 S2): the bodies of condition-raised, condition-changed and condition-cleared read the
|
||||
@@ -67,56 +65,6 @@ func TestTheControllersConditionEventsReadUnchanged(t *testing.T) {
|
||||
// take-over key the readers follow.
|
||||
func TestTheHealthLayoutsHoldTheKeepersRules(t *testing.T) {
|
||||
at := time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC)
|
||||
// ADR 0571 clause 3: a reading keeps its seven-day peak and is written only when it changes; a partial size is no peak.
|
||||
var prev *Reading
|
||||
for i, step := range []struct {
|
||||
day, size int64
|
||||
partial, wrote bool
|
||||
peak int64
|
||||
}{{20, 100, false, true, 100}, {20, 100, false, false, 100}, {21, 300, false, true, 300}, {22, 50, false, true, 300},
|
||||
{23, 9000, true, true, 300}, {29, 10, false, true, 50}, {30, 10, false, true, 10}} {
|
||||
now := time.Date(2026, 10, int(step.day), 1, 0, 0, 0, time.UTC)
|
||||
size := step.size
|
||||
m := Measured{SizeBytes: &size, MeasuredAt: &now, Precision: "exact"}
|
||||
if step.partial {
|
||||
m.Precision = "partial: stopped"
|
||||
}
|
||||
r, wrote := NextReading(prev, "ace", catalogue.PlacedDataItem{Module: "pg", ID: "data", Path: "/srv/pg"}, m, now)
|
||||
raw, _ := json.Marshal(r)
|
||||
back, err := DecodeReading(raw)
|
||||
if wrote != step.wrote || r.Peak == nil || *r.Peak != step.peak || err != nil || back.ID != "data" {
|
||||
t.Fatalf("reading %d: wrote %v, peak %v, read back %v", i, wrote, r.Peak, err)
|
||||
}
|
||||
prev = &r
|
||||
}
|
||||
// The first reading sets FirstSeen and every one after keeps it, as it keeps a retirement; an older reading reads as before.
|
||||
if first := time.Date(2026, 10, 20, 1, 0, 0, 0, time.UTC); !prev.FirstSeen.Equal(first) {
|
||||
t.Fatalf("first seen %s, not %s", prev.FirstSeen, first)
|
||||
}
|
||||
retired := *prev
|
||||
retired.RetiredAt, retired.RetiredWhy = at, "its module left ace"
|
||||
if r, _ := NextReading(&retired, "ace", retired.PlacedDataItem, retired.Measured, at.Add(time.Hour)); !r.RetiredAt.Equal(at) ||
|
||||
r.RetiredWhy != "its module left ace" || !r.FirstSeen.Equal(prev.FirstSeen) {
|
||||
t.Fatalf("a retired reading moved on as %+v", r)
|
||||
}
|
||||
if old, err := DecodeReading([]byte(`{"v":1,"node":"ace","module":"pg","item":"data"}`)); err != nil || !old.FirstSeen.IsZero() {
|
||||
t.Fatalf("a reading of before read as %+v: %v", old, err)
|
||||
}
|
||||
if raw, _ := json.Marshal(Reading{V: ReadingVersion}); strings.Contains(string(raw), "first_seen") || strings.Contains(string(raw), "retired") {
|
||||
t.Fatalf("zero fields were written: %s", raw)
|
||||
}
|
||||
if ReadingKey("ace", "pg", "data") != "ace.pg.data" || ReadingsBucket != "disk-load_readings" {
|
||||
t.Fatal("the readings' key or bucket moved")
|
||||
}
|
||||
// And restic's last good night per item, keyed as a reading.
|
||||
goodRow, _ := json.Marshal(GoodRow{V: GoodVersion, Node: "ace", Module: "pg", Item: "data", CoveredBy: "/srv/pg", LastGood: at})
|
||||
if g, err := DecodeGoodRow(goodRow); err != nil || !g.LastGood.Equal(at) || GoodKey("ace", "pg", "data") != "ace.pg.data" ||
|
||||
GoodBucket != "restic_good" || GoodRead != "restic.good" {
|
||||
t.Fatalf("a last good night read back as %+v: %v", g, err)
|
||||
}
|
||||
if _, err := DecodeGoodRow([]byte(`{"v":2}`)); err == nil {
|
||||
t.Fatal("a last good night of another version was read")
|
||||
}
|
||||
good := Finding{Key: "provider.keycloak.ace.mesh-mail.failing", Kind: "provider-failing", Ref: "failing",
|
||||
Subject: Subject{Scope: ScopeProvider, ID: "keycloak.ace.mesh-mail", Machine: "ace"},
|
||||
Severity: Warning, Summary: "keycloak on ace fails mesh-mail", Source: "S8", Observer: WatchmanModule, At: at}
|
||||
|
||||
@@ -1,176 +0,0 @@
|
||||
package health
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// **Disk usage readings and peaks per data item** (novox/hq ADR 0571 clause 3, ADR 0233 §21, §27, §34): disk-load, on every
|
||||
// node, measures each data item the modules of its node declare (catalogue.PlacedDataItem, from compose/frombus.DataItems)
|
||||
// and keeps the newest reading and the item's largest size in seven days as its own state `readings`, one key
|
||||
// `<node>.<module>.<item>`, written only when it changes. The node-backup holder, mounts, nfs-server and the self-check's
|
||||
// probes read it there; the controller keeps none of it. Moved here from mesh-catalog modules/disk-load (b4c237d2).
|
||||
const (
|
||||
DiskLoadModule = "disk-load"
|
||||
ReadingsState = "readings"
|
||||
ReadingsBucket = DiskLoadModule + "_" + ReadingsState
|
||||
ReadingsRead = DiskLoadModule + "." + ReadingsState
|
||||
)
|
||||
|
||||
// ReadingVersion is the version of a reading's layout.
|
||||
const ReadingVersion = 1
|
||||
|
||||
// PeakWindow is how far back the peak looks (ADR 0233 §34).
|
||||
const PeakWindow = 7 * 24 * time.Hour
|
||||
|
||||
// Redundancy is the redundant storage an item is on, as read: zfs, md or btrfs, its pool, array or filesystem, and its
|
||||
// health (nil when it could not be read).
|
||||
type Redundancy struct {
|
||||
Kind string `json:"kind"`
|
||||
Where string `json:"where"`
|
||||
Healthy *bool `json:"healthy"`
|
||||
Said string `json:"said"`
|
||||
}
|
||||
|
||||
// Measured is what one measurement found. Precision says what the size is: "exact", "dataset <name>: its whole size",
|
||||
// "partial: ..." (a lower bound, never compared), or "no size: ...".
|
||||
type Measured struct {
|
||||
SizeBytes *int64 `json:"size_bytes,omitempty"`
|
||||
LastWrite *time.Time `json:"last_write,omitempty"`
|
||||
MeasuredAt *time.Time `json:"measured_at,omitempty"`
|
||||
Precision string `json:"precision,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Redundancy *Redundancy `json:"redundancy,omitempty"`
|
||||
}
|
||||
|
||||
// Partial says the size is only a lower bound, which a peak is never made of.
|
||||
func (m Measured) Partial() bool { return strings.HasPrefix(m.Precision, "partial") }
|
||||
|
||||
// DaySize is the largest size measured on one UTC day.
|
||||
type DaySize struct {
|
||||
Day string `json:"day"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
}
|
||||
|
||||
// Reading is the value of one key: the item as placed, what was measured, and the largest size in the seven days up to
|
||||
// it (Peak, measured on PeakAt), read from the day maxima Days.
|
||||
type Reading struct {
|
||||
V int `json:"v"`
|
||||
Node string `json:"node"`
|
||||
catalogue.PlacedDataItem
|
||||
Measured
|
||||
Peak *int64 `json:"peak_bytes,omitempty"`
|
||||
PeakAt string `json:"peak_day,omitempty"`
|
||||
Days []DaySize `json:"days,omitempty"`
|
||||
// FirstSeen is when the item was first measured on this node, kept by every reading after; RetiredAt and RetiredWhy
|
||||
// say its module left the node, written by whoever retires it and carried on unchanged. Each is left out while zero,
|
||||
// so a reading written before reads as it did.
|
||||
FirstSeen time.Time `json:"first_seen,omitzero"`
|
||||
RetiredAt time.Time `json:"retired_at,omitzero"`
|
||||
RetiredWhy string `json:"retired_why,omitempty"`
|
||||
}
|
||||
|
||||
// ReadingKey is the key of an item on a node: `<node>.<module>.<item>`.
|
||||
func ReadingKey(node, module, item string) string { return node + "." + module + "." + item }
|
||||
|
||||
// DecodeReading reads a key's value; it refuses another version.
|
||||
func DecodeReading(raw []byte) (Reading, error) {
|
||||
var r Reading
|
||||
if err := json.Unmarshal(raw, &r); err != nil {
|
||||
return Reading{}, fmt.Errorf("health: a reading is not readable: %w", err)
|
||||
}
|
||||
if r.V != ReadingVersion {
|
||||
return Reading{}, fmt.Errorf("health: a reading of version %d, and this reads %d", r.V, ReadingVersion)
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// NextReading is the reading after a measurement, and whether it differs from the one before in anything but the time it
|
||||
// was taken: a reading that did not change is not written. A measurement with no size (an error, a shallow item) carries
|
||||
// the last size's days on, so the peak outlives one failed measurement.
|
||||
func NextReading(prev *Reading, node string, it catalogue.PlacedDataItem, m Measured, now time.Time) (Reading, bool) {
|
||||
r := Reading{V: ReadingVersion, Node: node, PlacedDataItem: it, Measured: m, FirstSeen: now.UTC()}
|
||||
var days []DaySize
|
||||
if prev != nil {
|
||||
days = prev.Days
|
||||
r.RetiredAt, r.RetiredWhy = prev.RetiredAt, prev.RetiredWhy
|
||||
if !prev.FirstSeen.IsZero() {
|
||||
r.FirstSeen = prev.FirstSeen
|
||||
}
|
||||
}
|
||||
today := now.UTC().Format("2006-01-02")
|
||||
cut := now.UTC().Add(-PeakWindow).Format("2006-01-02")
|
||||
var kept []DaySize
|
||||
todays := int64(-1)
|
||||
for _, d := range days {
|
||||
switch {
|
||||
case d.Day == today:
|
||||
todays = d.Bytes
|
||||
case d.Day >= cut:
|
||||
kept = append(kept, d)
|
||||
}
|
||||
}
|
||||
if m.SizeBytes != nil && !m.Partial() && *m.SizeBytes > todays {
|
||||
todays = *m.SizeBytes
|
||||
}
|
||||
if todays >= 0 {
|
||||
kept = append(kept, DaySize{Day: today, Bytes: todays})
|
||||
}
|
||||
r.Days = kept
|
||||
for _, d := range kept {
|
||||
if r.Peak == nil || d.Bytes >= *r.Peak {
|
||||
b := d.Bytes
|
||||
r.Peak, r.PeakAt = &b, d.Day
|
||||
}
|
||||
}
|
||||
if prev == nil {
|
||||
return r, true
|
||||
}
|
||||
a, b := *prev, r
|
||||
a.MeasuredAt, b.MeasuredAt = nil, nil
|
||||
return r, !reflect.DeepEqual(a, b)
|
||||
}
|
||||
|
||||
// **Each item's last good night** (novox/hq ADR 0571 clause 3, ADR 0564 §6; the lead, 2026-10-29): the node-backup holder
|
||||
// restic keeps, from its tools bundle through the runtime on the night's file event (its night's process runs as the
|
||||
// operator's account and holds no bus account, ADR 0436 §1-2), its own state `good`, one key `<node>.<module>.<item>` for every item a module
|
||||
// declares that a backup path covers, written when it changes: the item, the path whose snapshot keeps it and the newest
|
||||
// good night of its module (zero before the first). The self-check's probe of a stale backup reads it there. Moved here
|
||||
// from mesh-catalog modules/restic (e5ef3bed).
|
||||
const (
|
||||
ResticModule = "restic"
|
||||
GoodState = "good"
|
||||
GoodBucket = ResticModule + "_" + GoodState
|
||||
GoodRead = ResticModule + "." + GoodState
|
||||
GoodVersion = 1
|
||||
)
|
||||
|
||||
// GoodRow is one key of restic's `good`.
|
||||
type GoodRow struct {
|
||||
V int `json:"v"`
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Item string `json:"item"`
|
||||
CoveredBy string `json:"covered_by"`
|
||||
LastGood time.Time `json:"last_good,omitzero"`
|
||||
}
|
||||
|
||||
// GoodKey is a row's key: `<node>.<module>.<item>`, as a reading's.
|
||||
func GoodKey(node, module, item string) string { return ReadingKey(node, module, item) }
|
||||
|
||||
// DecodeGoodRow reads a row; it refuses another version.
|
||||
func DecodeGoodRow(raw []byte) (GoodRow, error) {
|
||||
var g GoodRow
|
||||
if err := json.Unmarshal(raw, &g); err != nil {
|
||||
return GoodRow{}, fmt.Errorf("health: a last good night is not readable: %w", err)
|
||||
}
|
||||
if g.V != GoodVersion {
|
||||
return GoodRow{}, fmt.Errorf("health: a last good night of version %d, and this reads %d", g.V, GoodVersion)
|
||||
}
|
||||
return g, nil
|
||||
}
|
||||
Reference in New Issue
Block a user