diff --git a/go/health/condition_test.go b/go/health/condition_test.go index f6decea..05269d9 100644 --- a/go/health/condition_test.go +++ b/go/health/condition_test.go @@ -7,8 +7,6 @@ import ( "strings" "testing" "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" ) // ADR 0564 clause 3 (issue 1952 S2): the bodies of condition-raised, condition-changed and condition-cleared read the @@ -67,56 +65,6 @@ func TestTheControllersConditionEventsReadUnchanged(t *testing.T) { // take-over key the readers follow. func TestTheHealthLayoutsHoldTheKeepersRules(t *testing.T) { at := time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC) - // ADR 0571 clause 3: a reading keeps its seven-day peak and is written only when it changes; a partial size is no peak. - var prev *Reading - for i, step := range []struct { - day, size int64 - partial, wrote bool - peak int64 - }{{20, 100, false, true, 100}, {20, 100, false, false, 100}, {21, 300, false, true, 300}, {22, 50, false, true, 300}, - {23, 9000, true, true, 300}, {29, 10, false, true, 50}, {30, 10, false, true, 10}} { - now := time.Date(2026, 10, int(step.day), 1, 0, 0, 0, time.UTC) - size := step.size - m := Measured{SizeBytes: &size, MeasuredAt: &now, Precision: "exact"} - if step.partial { - m.Precision = "partial: stopped" - } - r, wrote := NextReading(prev, "ace", catalogue.PlacedDataItem{Module: "pg", ID: "data", Path: "/srv/pg"}, m, now) - raw, _ := json.Marshal(r) - back, err := DecodeReading(raw) - if wrote != step.wrote || r.Peak == nil || *r.Peak != step.peak || err != nil || back.ID != "data" { - t.Fatalf("reading %d: wrote %v, peak %v, read back %v", i, wrote, r.Peak, err) - } - prev = &r - } - // The first reading sets FirstSeen and every one after keeps it, as it keeps a retirement; an older reading reads as before. - if first := time.Date(2026, 10, 20, 1, 0, 0, 0, time.UTC); !prev.FirstSeen.Equal(first) { - t.Fatalf("first seen %s, not %s", prev.FirstSeen, first) - } - retired := *prev - retired.RetiredAt, retired.RetiredWhy = at, "its module left ace" - if r, _ := NextReading(&retired, "ace", retired.PlacedDataItem, retired.Measured, at.Add(time.Hour)); !r.RetiredAt.Equal(at) || - r.RetiredWhy != "its module left ace" || !r.FirstSeen.Equal(prev.FirstSeen) { - t.Fatalf("a retired reading moved on as %+v", r) - } - if old, err := DecodeReading([]byte(`{"v":1,"node":"ace","module":"pg","item":"data"}`)); err != nil || !old.FirstSeen.IsZero() { - t.Fatalf("a reading of before read as %+v: %v", old, err) - } - if raw, _ := json.Marshal(Reading{V: ReadingVersion}); strings.Contains(string(raw), "first_seen") || strings.Contains(string(raw), "retired") { - t.Fatalf("zero fields were written: %s", raw) - } - if ReadingKey("ace", "pg", "data") != "ace.pg.data" || ReadingsBucket != "disk-load_readings" { - t.Fatal("the readings' key or bucket moved") - } - // And restic's last good night per item, keyed as a reading. - goodRow, _ := json.Marshal(GoodRow{V: GoodVersion, Node: "ace", Module: "pg", Item: "data", CoveredBy: "/srv/pg", LastGood: at}) - if g, err := DecodeGoodRow(goodRow); err != nil || !g.LastGood.Equal(at) || GoodKey("ace", "pg", "data") != "ace.pg.data" || - GoodBucket != "restic_good" || GoodRead != "restic.good" { - t.Fatalf("a last good night read back as %+v: %v", g, err) - } - if _, err := DecodeGoodRow([]byte(`{"v":2}`)); err == nil { - t.Fatal("a last good night of another version was read") - } good := Finding{Key: "provider.keycloak.ace.mesh-mail.failing", Kind: "provider-failing", Ref: "failing", Subject: Subject{Scope: ScopeProvider, ID: "keycloak.ace.mesh-mail", Machine: "ace"}, Severity: Warning, Summary: "keycloak on ace fails mesh-mail", Source: "S8", Observer: WatchmanModule, At: at} diff --git a/go/health/readings.go b/go/health/readings.go deleted file mode 100644 index e02c960..0000000 --- a/go/health/readings.go +++ /dev/null @@ -1,176 +0,0 @@ -package health - -import ( - "encoding/json" - "fmt" - "reflect" - "strings" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" -) - -// **Disk usage readings and peaks per data item** (novox/hq ADR 0571 clause 3, ADR 0233 §21, §27, §34): disk-load, on every -// node, measures each data item the modules of its node declare (catalogue.PlacedDataItem, from compose/frombus.DataItems) -// and keeps the newest reading and the item's largest size in seven days as its own state `readings`, one key -// `..`, written only when it changes. The node-backup holder, mounts, nfs-server and the self-check's -// probes read it there; the controller keeps none of it. Moved here from mesh-catalog modules/disk-load (b4c237d2). -const ( - DiskLoadModule = "disk-load" - ReadingsState = "readings" - ReadingsBucket = DiskLoadModule + "_" + ReadingsState - ReadingsRead = DiskLoadModule + "." + ReadingsState -) - -// ReadingVersion is the version of a reading's layout. -const ReadingVersion = 1 - -// PeakWindow is how far back the peak looks (ADR 0233 §34). -const PeakWindow = 7 * 24 * time.Hour - -// Redundancy is the redundant storage an item is on, as read: zfs, md or btrfs, its pool, array or filesystem, and its -// health (nil when it could not be read). -type Redundancy struct { - Kind string `json:"kind"` - Where string `json:"where"` - Healthy *bool `json:"healthy"` - Said string `json:"said"` -} - -// Measured is what one measurement found. Precision says what the size is: "exact", "dataset : its whole size", -// "partial: ..." (a lower bound, never compared), or "no size: ...". -type Measured struct { - SizeBytes *int64 `json:"size_bytes,omitempty"` - LastWrite *time.Time `json:"last_write,omitempty"` - MeasuredAt *time.Time `json:"measured_at,omitempty"` - Precision string `json:"precision,omitempty"` - Error string `json:"error,omitempty"` - Redundancy *Redundancy `json:"redundancy,omitempty"` -} - -// Partial says the size is only a lower bound, which a peak is never made of. -func (m Measured) Partial() bool { return strings.HasPrefix(m.Precision, "partial") } - -// DaySize is the largest size measured on one UTC day. -type DaySize struct { - Day string `json:"day"` - Bytes int64 `json:"bytes"` -} - -// Reading is the value of one key: the item as placed, what was measured, and the largest size in the seven days up to -// it (Peak, measured on PeakAt), read from the day maxima Days. -type Reading struct { - V int `json:"v"` - Node string `json:"node"` - catalogue.PlacedDataItem - Measured - Peak *int64 `json:"peak_bytes,omitempty"` - PeakAt string `json:"peak_day,omitempty"` - Days []DaySize `json:"days,omitempty"` - // FirstSeen is when the item was first measured on this node, kept by every reading after; RetiredAt and RetiredWhy - // say its module left the node, written by whoever retires it and carried on unchanged. Each is left out while zero, - // so a reading written before reads as it did. - FirstSeen time.Time `json:"first_seen,omitzero"` - RetiredAt time.Time `json:"retired_at,omitzero"` - RetiredWhy string `json:"retired_why,omitempty"` -} - -// ReadingKey is the key of an item on a node: `..`. -func ReadingKey(node, module, item string) string { return node + "." + module + "." + item } - -// DecodeReading reads a key's value; it refuses another version. -func DecodeReading(raw []byte) (Reading, error) { - var r Reading - if err := json.Unmarshal(raw, &r); err != nil { - return Reading{}, fmt.Errorf("health: a reading is not readable: %w", err) - } - if r.V != ReadingVersion { - return Reading{}, fmt.Errorf("health: a reading of version %d, and this reads %d", r.V, ReadingVersion) - } - return r, nil -} - -// NextReading is the reading after a measurement, and whether it differs from the one before in anything but the time it -// was taken: a reading that did not change is not written. A measurement with no size (an error, a shallow item) carries -// the last size's days on, so the peak outlives one failed measurement. -func NextReading(prev *Reading, node string, it catalogue.PlacedDataItem, m Measured, now time.Time) (Reading, bool) { - r := Reading{V: ReadingVersion, Node: node, PlacedDataItem: it, Measured: m, FirstSeen: now.UTC()} - var days []DaySize - if prev != nil { - days = prev.Days - r.RetiredAt, r.RetiredWhy = prev.RetiredAt, prev.RetiredWhy - if !prev.FirstSeen.IsZero() { - r.FirstSeen = prev.FirstSeen - } - } - today := now.UTC().Format("2006-01-02") - cut := now.UTC().Add(-PeakWindow).Format("2006-01-02") - var kept []DaySize - todays := int64(-1) - for _, d := range days { - switch { - case d.Day == today: - todays = d.Bytes - case d.Day >= cut: - kept = append(kept, d) - } - } - if m.SizeBytes != nil && !m.Partial() && *m.SizeBytes > todays { - todays = *m.SizeBytes - } - if todays >= 0 { - kept = append(kept, DaySize{Day: today, Bytes: todays}) - } - r.Days = kept - for _, d := range kept { - if r.Peak == nil || d.Bytes >= *r.Peak { - b := d.Bytes - r.Peak, r.PeakAt = &b, d.Day - } - } - if prev == nil { - return r, true - } - a, b := *prev, r - a.MeasuredAt, b.MeasuredAt = nil, nil - return r, !reflect.DeepEqual(a, b) -} - -// **Each item's last good night** (novox/hq ADR 0571 clause 3, ADR 0564 §6; the lead, 2026-10-29): the node-backup holder -// restic keeps, from its tools bundle through the runtime on the night's file event (its night's process runs as the -// operator's account and holds no bus account, ADR 0436 §1-2), its own state `good`, one key `..` for every item a module -// declares that a backup path covers, written when it changes: the item, the path whose snapshot keeps it and the newest -// good night of its module (zero before the first). The self-check's probe of a stale backup reads it there. Moved here -// from mesh-catalog modules/restic (e5ef3bed). -const ( - ResticModule = "restic" - GoodState = "good" - GoodBucket = ResticModule + "_" + GoodState - GoodRead = ResticModule + "." + GoodState - GoodVersion = 1 -) - -// GoodRow is one key of restic's `good`. -type GoodRow struct { - V int `json:"v"` - Node string `json:"node"` - Module string `json:"module"` - Item string `json:"item"` - CoveredBy string `json:"covered_by"` - LastGood time.Time `json:"last_good,omitzero"` -} - -// GoodKey is a row's key: `..`, as a reading's. -func GoodKey(node, module, item string) string { return ReadingKey(node, module, item) } - -// DecodeGoodRow reads a row; it refuses another version. -func DecodeGoodRow(raw []byte) (GoodRow, error) { - var g GoodRow - if err := json.Unmarshal(raw, &g); err != nil { - return GoodRow{}, fmt.Errorf("health: a last good night is not readable: %w", err) - } - if g.V != GoodVersion { - return GoodRow{}, fmt.Errorf("health: a last good night of version %d, and this reads %d", g.V, GoodVersion) - } - return g, nil -}