health: readings.go (disk-load's readings, restic's last good night) leaves, held by disk-load's diskreadings and restic's backupgood (S-7 part, B9f; issue 1966)
Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i
This commit is contained in:
@@ -7,8 +7,6 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// ADR 0564 clause 3 (issue 1952 S2): the bodies of condition-raised, condition-changed and condition-cleared read the
|
// ADR 0564 clause 3 (issue 1952 S2): the bodies of condition-raised, condition-changed and condition-cleared read the
|
||||||
@@ -67,56 +65,6 @@ func TestTheControllersConditionEventsReadUnchanged(t *testing.T) {
|
|||||||
// take-over key the readers follow.
|
// take-over key the readers follow.
|
||||||
func TestTheHealthLayoutsHoldTheKeepersRules(t *testing.T) {
|
func TestTheHealthLayoutsHoldTheKeepersRules(t *testing.T) {
|
||||||
at := time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC)
|
at := time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC)
|
||||||
// ADR 0571 clause 3: a reading keeps its seven-day peak and is written only when it changes; a partial size is no peak.
|
|
||||||
var prev *Reading
|
|
||||||
for i, step := range []struct {
|
|
||||||
day, size int64
|
|
||||||
partial, wrote bool
|
|
||||||
peak int64
|
|
||||||
}{{20, 100, false, true, 100}, {20, 100, false, false, 100}, {21, 300, false, true, 300}, {22, 50, false, true, 300},
|
|
||||||
{23, 9000, true, true, 300}, {29, 10, false, true, 50}, {30, 10, false, true, 10}} {
|
|
||||||
now := time.Date(2026, 10, int(step.day), 1, 0, 0, 0, time.UTC)
|
|
||||||
size := step.size
|
|
||||||
m := Measured{SizeBytes: &size, MeasuredAt: &now, Precision: "exact"}
|
|
||||||
if step.partial {
|
|
||||||
m.Precision = "partial: stopped"
|
|
||||||
}
|
|
||||||
r, wrote := NextReading(prev, "ace", catalogue.PlacedDataItem{Module: "pg", ID: "data", Path: "/srv/pg"}, m, now)
|
|
||||||
raw, _ := json.Marshal(r)
|
|
||||||
back, err := DecodeReading(raw)
|
|
||||||
if wrote != step.wrote || r.Peak == nil || *r.Peak != step.peak || err != nil || back.ID != "data" {
|
|
||||||
t.Fatalf("reading %d: wrote %v, peak %v, read back %v", i, wrote, r.Peak, err)
|
|
||||||
}
|
|
||||||
prev = &r
|
|
||||||
}
|
|
||||||
// The first reading sets FirstSeen and every one after keeps it, as it keeps a retirement; an older reading reads as before.
|
|
||||||
if first := time.Date(2026, 10, 20, 1, 0, 0, 0, time.UTC); !prev.FirstSeen.Equal(first) {
|
|
||||||
t.Fatalf("first seen %s, not %s", prev.FirstSeen, first)
|
|
||||||
}
|
|
||||||
retired := *prev
|
|
||||||
retired.RetiredAt, retired.RetiredWhy = at, "its module left ace"
|
|
||||||
if r, _ := NextReading(&retired, "ace", retired.PlacedDataItem, retired.Measured, at.Add(time.Hour)); !r.RetiredAt.Equal(at) ||
|
|
||||||
r.RetiredWhy != "its module left ace" || !r.FirstSeen.Equal(prev.FirstSeen) {
|
|
||||||
t.Fatalf("a retired reading moved on as %+v", r)
|
|
||||||
}
|
|
||||||
if old, err := DecodeReading([]byte(`{"v":1,"node":"ace","module":"pg","item":"data"}`)); err != nil || !old.FirstSeen.IsZero() {
|
|
||||||
t.Fatalf("a reading of before read as %+v: %v", old, err)
|
|
||||||
}
|
|
||||||
if raw, _ := json.Marshal(Reading{V: ReadingVersion}); strings.Contains(string(raw), "first_seen") || strings.Contains(string(raw), "retired") {
|
|
||||||
t.Fatalf("zero fields were written: %s", raw)
|
|
||||||
}
|
|
||||||
if ReadingKey("ace", "pg", "data") != "ace.pg.data" || ReadingsBucket != "disk-load_readings" {
|
|
||||||
t.Fatal("the readings' key or bucket moved")
|
|
||||||
}
|
|
||||||
// And restic's last good night per item, keyed as a reading.
|
|
||||||
goodRow, _ := json.Marshal(GoodRow{V: GoodVersion, Node: "ace", Module: "pg", Item: "data", CoveredBy: "/srv/pg", LastGood: at})
|
|
||||||
if g, err := DecodeGoodRow(goodRow); err != nil || !g.LastGood.Equal(at) || GoodKey("ace", "pg", "data") != "ace.pg.data" ||
|
|
||||||
GoodBucket != "restic_good" || GoodRead != "restic.good" {
|
|
||||||
t.Fatalf("a last good night read back as %+v: %v", g, err)
|
|
||||||
}
|
|
||||||
if _, err := DecodeGoodRow([]byte(`{"v":2}`)); err == nil {
|
|
||||||
t.Fatal("a last good night of another version was read")
|
|
||||||
}
|
|
||||||
good := Finding{Key: "provider.keycloak.ace.mesh-mail.failing", Kind: "provider-failing", Ref: "failing",
|
good := Finding{Key: "provider.keycloak.ace.mesh-mail.failing", Kind: "provider-failing", Ref: "failing",
|
||||||
Subject: Subject{Scope: ScopeProvider, ID: "keycloak.ace.mesh-mail", Machine: "ace"},
|
Subject: Subject{Scope: ScopeProvider, ID: "keycloak.ace.mesh-mail", Machine: "ace"},
|
||||||
Severity: Warning, Summary: "keycloak on ace fails mesh-mail", Source: "S8", Observer: WatchmanModule, At: at}
|
Severity: Warning, Summary: "keycloak on ace fails mesh-mail", Source: "S8", Observer: WatchmanModule, At: at}
|
||||||
|
|||||||
@@ -1,176 +0,0 @@
|
|||||||
package health
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"reflect"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
)
|
|
||||||
|
|
||||||
// **Disk usage readings and peaks per data item** (novox/hq ADR 0571 clause 3, ADR 0233 §21, §27, §34): disk-load, on every
|
|
||||||
// node, measures each data item the modules of its node declare (catalogue.PlacedDataItem, from compose/frombus.DataItems)
|
|
||||||
// and keeps the newest reading and the item's largest size in seven days as its own state `readings`, one key
|
|
||||||
// `<node>.<module>.<item>`, written only when it changes. The node-backup holder, mounts, nfs-server and the self-check's
|
|
||||||
// probes read it there; the controller keeps none of it. Moved here from mesh-catalog modules/disk-load (b4c237d2).
|
|
||||||
const (
|
|
||||||
DiskLoadModule = "disk-load"
|
|
||||||
ReadingsState = "readings"
|
|
||||||
ReadingsBucket = DiskLoadModule + "_" + ReadingsState
|
|
||||||
ReadingsRead = DiskLoadModule + "." + ReadingsState
|
|
||||||
)
|
|
||||||
|
|
||||||
// ReadingVersion is the version of a reading's layout.
|
|
||||||
const ReadingVersion = 1
|
|
||||||
|
|
||||||
// PeakWindow is how far back the peak looks (ADR 0233 §34).
|
|
||||||
const PeakWindow = 7 * 24 * time.Hour
|
|
||||||
|
|
||||||
// Redundancy is the redundant storage an item is on, as read: zfs, md or btrfs, its pool, array or filesystem, and its
|
|
||||||
// health (nil when it could not be read).
|
|
||||||
type Redundancy struct {
|
|
||||||
Kind string `json:"kind"`
|
|
||||||
Where string `json:"where"`
|
|
||||||
Healthy *bool `json:"healthy"`
|
|
||||||
Said string `json:"said"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// Measured is what one measurement found. Precision says what the size is: "exact", "dataset <name>: its whole size",
|
|
||||||
// "partial: ..." (a lower bound, never compared), or "no size: ...".
|
|
||||||
type Measured struct {
|
|
||||||
SizeBytes *int64 `json:"size_bytes,omitempty"`
|
|
||||||
LastWrite *time.Time `json:"last_write,omitempty"`
|
|
||||||
MeasuredAt *time.Time `json:"measured_at,omitempty"`
|
|
||||||
Precision string `json:"precision,omitempty"`
|
|
||||||
Error string `json:"error,omitempty"`
|
|
||||||
Redundancy *Redundancy `json:"redundancy,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// Partial says the size is only a lower bound, which a peak is never made of.
|
|
||||||
func (m Measured) Partial() bool { return strings.HasPrefix(m.Precision, "partial") }
|
|
||||||
|
|
||||||
// DaySize is the largest size measured on one UTC day.
|
|
||||||
type DaySize struct {
|
|
||||||
Day string `json:"day"`
|
|
||||||
Bytes int64 `json:"bytes"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// Reading is the value of one key: the item as placed, what was measured, and the largest size in the seven days up to
|
|
||||||
// it (Peak, measured on PeakAt), read from the day maxima Days.
|
|
||||||
type Reading struct {
|
|
||||||
V int `json:"v"`
|
|
||||||
Node string `json:"node"`
|
|
||||||
catalogue.PlacedDataItem
|
|
||||||
Measured
|
|
||||||
Peak *int64 `json:"peak_bytes,omitempty"`
|
|
||||||
PeakAt string `json:"peak_day,omitempty"`
|
|
||||||
Days []DaySize `json:"days,omitempty"`
|
|
||||||
// FirstSeen is when the item was first measured on this node, kept by every reading after; RetiredAt and RetiredWhy
|
|
||||||
// say its module left the node, written by whoever retires it and carried on unchanged. Each is left out while zero,
|
|
||||||
// so a reading written before reads as it did.
|
|
||||||
FirstSeen time.Time `json:"first_seen,omitzero"`
|
|
||||||
RetiredAt time.Time `json:"retired_at,omitzero"`
|
|
||||||
RetiredWhy string `json:"retired_why,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// ReadingKey is the key of an item on a node: `<node>.<module>.<item>`.
|
|
||||||
func ReadingKey(node, module, item string) string { return node + "." + module + "." + item }
|
|
||||||
|
|
||||||
// DecodeReading reads a key's value; it refuses another version.
|
|
||||||
func DecodeReading(raw []byte) (Reading, error) {
|
|
||||||
var r Reading
|
|
||||||
if err := json.Unmarshal(raw, &r); err != nil {
|
|
||||||
return Reading{}, fmt.Errorf("health: a reading is not readable: %w", err)
|
|
||||||
}
|
|
||||||
if r.V != ReadingVersion {
|
|
||||||
return Reading{}, fmt.Errorf("health: a reading of version %d, and this reads %d", r.V, ReadingVersion)
|
|
||||||
}
|
|
||||||
return r, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// NextReading is the reading after a measurement, and whether it differs from the one before in anything but the time it
|
|
||||||
// was taken: a reading that did not change is not written. A measurement with no size (an error, a shallow item) carries
|
|
||||||
// the last size's days on, so the peak outlives one failed measurement.
|
|
||||||
func NextReading(prev *Reading, node string, it catalogue.PlacedDataItem, m Measured, now time.Time) (Reading, bool) {
|
|
||||||
r := Reading{V: ReadingVersion, Node: node, PlacedDataItem: it, Measured: m, FirstSeen: now.UTC()}
|
|
||||||
var days []DaySize
|
|
||||||
if prev != nil {
|
|
||||||
days = prev.Days
|
|
||||||
r.RetiredAt, r.RetiredWhy = prev.RetiredAt, prev.RetiredWhy
|
|
||||||
if !prev.FirstSeen.IsZero() {
|
|
||||||
r.FirstSeen = prev.FirstSeen
|
|
||||||
}
|
|
||||||
}
|
|
||||||
today := now.UTC().Format("2006-01-02")
|
|
||||||
cut := now.UTC().Add(-PeakWindow).Format("2006-01-02")
|
|
||||||
var kept []DaySize
|
|
||||||
todays := int64(-1)
|
|
||||||
for _, d := range days {
|
|
||||||
switch {
|
|
||||||
case d.Day == today:
|
|
||||||
todays = d.Bytes
|
|
||||||
case d.Day >= cut:
|
|
||||||
kept = append(kept, d)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if m.SizeBytes != nil && !m.Partial() && *m.SizeBytes > todays {
|
|
||||||
todays = *m.SizeBytes
|
|
||||||
}
|
|
||||||
if todays >= 0 {
|
|
||||||
kept = append(kept, DaySize{Day: today, Bytes: todays})
|
|
||||||
}
|
|
||||||
r.Days = kept
|
|
||||||
for _, d := range kept {
|
|
||||||
if r.Peak == nil || d.Bytes >= *r.Peak {
|
|
||||||
b := d.Bytes
|
|
||||||
r.Peak, r.PeakAt = &b, d.Day
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if prev == nil {
|
|
||||||
return r, true
|
|
||||||
}
|
|
||||||
a, b := *prev, r
|
|
||||||
a.MeasuredAt, b.MeasuredAt = nil, nil
|
|
||||||
return r, !reflect.DeepEqual(a, b)
|
|
||||||
}
|
|
||||||
|
|
||||||
// **Each item's last good night** (novox/hq ADR 0571 clause 3, ADR 0564 §6; the lead, 2026-10-29): the node-backup holder
|
|
||||||
// restic keeps, from its tools bundle through the runtime on the night's file event (its night's process runs as the
|
|
||||||
// operator's account and holds no bus account, ADR 0436 §1-2), its own state `good`, one key `<node>.<module>.<item>` for every item a module
|
|
||||||
// declares that a backup path covers, written when it changes: the item, the path whose snapshot keeps it and the newest
|
|
||||||
// good night of its module (zero before the first). The self-check's probe of a stale backup reads it there. Moved here
|
|
||||||
// from mesh-catalog modules/restic (e5ef3bed).
|
|
||||||
const (
|
|
||||||
ResticModule = "restic"
|
|
||||||
GoodState = "good"
|
|
||||||
GoodBucket = ResticModule + "_" + GoodState
|
|
||||||
GoodRead = ResticModule + "." + GoodState
|
|
||||||
GoodVersion = 1
|
|
||||||
)
|
|
||||||
|
|
||||||
// GoodRow is one key of restic's `good`.
|
|
||||||
type GoodRow struct {
|
|
||||||
V int `json:"v"`
|
|
||||||
Node string `json:"node"`
|
|
||||||
Module string `json:"module"`
|
|
||||||
Item string `json:"item"`
|
|
||||||
CoveredBy string `json:"covered_by"`
|
|
||||||
LastGood time.Time `json:"last_good,omitzero"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// GoodKey is a row's key: `<node>.<module>.<item>`, as a reading's.
|
|
||||||
func GoodKey(node, module, item string) string { return ReadingKey(node, module, item) }
|
|
||||||
|
|
||||||
// DecodeGoodRow reads a row; it refuses another version.
|
|
||||||
func DecodeGoodRow(raw []byte) (GoodRow, error) {
|
|
||||||
var g GoodRow
|
|
||||||
if err := json.Unmarshal(raw, &g); err != nil {
|
|
||||||
return GoodRow{}, fmt.Errorf("health: a last good night is not readable: %w", err)
|
|
||||||
}
|
|
||||||
if g.V != GoodVersion {
|
|
||||||
return GoodRow{}, fmt.Errorf("health: a last good night of version %d, and this reads %d", g.V, GoodVersion)
|
|
||||||
}
|
|
||||||
return g, nil
|
|
||||||
}
|
|
||||||
Reference in New Issue
Block a user