novox/mesh-catalog
18
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Readings and last good nights are the owners' published packages: disk-load's diskreadings and restic's backupgood, taken by series (B5b Left 3; ADR 0634 §4; issue 1966)

Browse Source 
Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i
This commit is contained in:
jschoubben
2026-11-05 14:48:10 +01:00
parent c7ae96414f
commit e0eecbf3c8
43 changed files with 565 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/disk-load/cmd/disk-load/data_test.go
+4 -4
View File
@@ -9,10 +9,10 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
)
// ADR 0577 clauses 2, 3 (and the disk-load check of clause 5): the items measured come from the node-engine's keys of
@@ -137,7 +137,7 @@ func TestTheItemsComeFromTheEnginesKeysAndARoundSaysHowFarItGot(t *testing.T) {
// A restart: oldie's key was deleted while the watch was down, so no delete arrives. Once the watch handed over its
// keys, oldie's reading is retired and pg's kept; an empty set retires nothing (the engine has not written yet).
_ = readings.Put("ace.oldie.x", health.Reading{V: health.ReadingVersion, Node: "ace",
_ = readings.Put("ace.oldie.x", diskreadings.Reading{V: diskreadings.ReadingVersion, Node: "ace",
PlacedDataItem: catalogue.PlacedDataItem{Module: "oldie", ID: "x", Path: "/srv/oldie"}})
for _, restart := range []struct {
name string
@@ -167,12 +167,12 @@ func TestTheItemsComeFromTheEnginesKeysAndARoundSaysHowFarItGot(t *testing.T) {
}
}
func readingOf(s *memStore, key string) *health.Reading {
func readingOf(s *memStore, key string) *diskreadings.Reading {
raw, _ := s.Get(key)
if raw == nil {
return nil
}
r, err := health.DecodeReading(raw)
r, err := diskreadings.DecodeReading(raw)
if err != nil {
return nil
}
modules/disk-load/cmd/disk-load/keeper.go
+18 -18
View File
@@ -6,9 +6,9 @@ import (
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
"os"
"os/exec"
"sort"
@@ -287,7 +287,7 @@ func (k *Keeper) MeasureModule(ctx context.Context, module string) {
// asked only on a change of that module's key (novox/hq ADR 0577 clause 2), never from a round. It is cleared when the
// item is declared again (measureOne).
func (k *Keeper) retireGone(module string, declared []catalogue.PlacedDataItem) (int, error) {
return k.retire(func(r health.Reading) bool { return r.Module == module }, declared)
return k.retire(func(r diskreadings.Reading) bool { return r.Module == module }, declared)
}
// RetireUnkeyed retires, once the watch handed over every current key, the readings of this node whose module has no
@@ -310,7 +310,7 @@ func (k *Keeper) RetireUnkeyed(ctx context.Context) {
for _, its := range items {
declared = append(declared, its...)
}
n, err := k.retire(func(r health.Reading) bool { _, keyed := items[r.Module]; return !keyed }, declared)
n, err := k.retire(func(r diskreadings.Reading) bool { _, keyed := items[r.Module]; return !keyed }, declared)
switch {
case err != nil:
k.Say("the readings of modules with no key are not all retired: %v", err)
@@ -320,14 +320,14 @@ func (k *Keeper) RetireUnkeyed(ctx context.Context) {
}
// retire marks retired every reading of this node that of picks and declared does not name.
func (k *Keeper) retire(of func(health.Reading) bool, declared []catalogue.PlacedDataItem) (int, error) {
func (k *Keeper) retire(of func(diskreadings.Reading) bool, declared []catalogue.PlacedDataItem) (int, error) {
keys, err := k.Store.Keys()
if err != nil {
return 0, err
}
declaredNow := map[string]bool{}
for _, it := range declared {
declaredNow[health.ReadingKey(k.Node, it.Module, it.ID)] = true
declaredNow[diskreadings.ReadingKey(k.Node, it.Module, it.ID)] = true
}
now := k.Measurer.Now().UTC()
n := 0
@@ -339,7 +339,7 @@ func (k *Keeper) retire(of func(health.Reading) bool, declared []catalogue.Place
if err != nil || len(raw) == 0 {
continue
}
r, err := health.DecodeReading(raw)
r, err := diskreadings.DecodeReading(raw)
if err != nil || r.Node != k.Node || !of(r) || !r.RetiredAt.IsZero() {
continue
}
@@ -357,14 +357,14 @@ func (k *Keeper) retire(of func(health.Reading) bool, declared []catalogue.Place
// when it changed. It answers whether it wrote the reading, what the measurement said failed ("" when it did not, or
// when the item was not measured now), and an error when the reading could not be read or kept.
func (k *Keeper) measureOne(ctx context.Context, it catalogue.PlacedDataItem, walks bool) (bool, string, error) {
key := health.ReadingKey(k.Node, it.Module, it.ID)
var prev *health.Reading
key := diskreadings.ReadingKey(k.Node, it.Module, it.ID)
var prev *diskreadings.Reading
raw, err := k.Store.Get(key)
if err != nil {
return false, "", err
}
if len(raw) > 0 {
if p, err := health.DecodeReading(raw); err == nil {
if p, err := diskreadings.DecodeReading(raw); err == nil {
prev = &p
}
}
@@ -374,7 +374,7 @@ func (k *Keeper) measureOne(ctx context.Context, it catalogue.PlacedDataItem, wa
return false, "", nil
}
m := k.Measurer.Measure(ctx, it)
next, changed := health.NextReading(prev, k.Node, it, m, now)
next, changed := diskreadings.NextReading(prev, k.Node, it, m, now)
if prev != nil && !prev.RetiredAt.IsZero() {
// Declared again: it is no longer retired.
next.RetiredAt, next.RetiredWhy, changed = time.Time{}, "", true
@@ -386,32 +386,32 @@ func (k *Keeper) measureOne(ctx context.Context, it catalogue.PlacedDataItem, wa
}
// MeasureNow measures one item now, walk or not, and writes its reading when it changed.
func (k *Keeper) MeasureNow(ctx context.Context, module, item string) (health.Reading, bool, error) {
func (k *Keeper) MeasureNow(ctx context.Context, module, item string) (diskreadings.Reading, bool, error) {
k.mu.Lock()
defer k.mu.Unlock()
items, err := k.Items(ctx)
if err != nil {
return health.Reading{}, false, err
return diskreadings.Reading{}, false, err
}
for _, it := range items[module] {
if it.ID != item {
continue
}
if it.Class == ClassCache {
return health.Reading{}, false, fmt.Errorf("%s.%s is a cache, which is listed and never measured", module, item)
return diskreadings.Reading{}, false, fmt.Errorf("%s.%s is a cache, which is listed and never measured", module, item)
}
w, _, err := k.measureOne(ctx, it, true)
if err != nil {
return health.Reading{}, false, err
return diskreadings.Reading{}, false, err
}
raw, err := k.Store.Get(health.ReadingKey(k.Node, module, item))
raw, err := k.Store.Get(diskreadings.ReadingKey(k.Node, module, item))
if err != nil {
return health.Reading{}, w, err
return diskreadings.Reading{}, w, err
}
r, err := health.DecodeReading(raw)
r, err := diskreadings.DecodeReading(raw)
return r, w, err
}
return health.Reading{}, false, fmt.Errorf("%s declares no data item %q on this machine", module, item)
return diskreadings.Reading{}, false, fmt.Errorf("%s declares no data item %q on this machine", module, item)
}
// Loop runs a round after a few minutes and then every `every`, with the walks of the night's round when it is the
modules/disk-load/cmd/disk-load/measure.go
+7 -7
View File
@@ -6,8 +6,8 @@ package main
import (
"context"
"fmt"
"git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/health"
"regexp"
"strconv"
"strings"
@@ -56,9 +56,9 @@ func (b Measurer) exists(ctx context.Context, path string) bool {
}
// Measure is one item, measured now, by its own method.
func (b Measurer) Measure(ctx context.Context, it catalogue.PlacedDataItem) health.Measured {
func (b Measurer) Measure(ctx context.Context, it catalogue.PlacedDataItem) diskreadings.Measured {
at := b.Now().UTC()
m := health.Measured{MeasuredAt: &at}
m := diskreadings.Measured{MeasuredAt: &at}
if !b.exists(ctx, it.Path) {
m.Error = it.Path + " does not exist"
zero := int64(0)
@@ -120,7 +120,7 @@ func (b Measurer) Measure(ctx context.Context, it catalogue.PlacedDataItem) heal
// redundancyOf is the redundant storage a path is on: a ZFS pool's health, an md array's members, a btrfs
// filesystem's error counters. Nil for storage with no redundancy it knows how to read.
func (b Measurer) redundancyOf(ctx context.Context, path string) *health.Redundancy {
func (b Measurer) redundancyOf(ctx context.Context, path string) *diskreadings.Redundancy {
out, err := b.Run(ctx, "findmnt", "-no", "SOURCE,FSTYPE", "--target", path)
if err != nil {
return nil
@@ -134,7 +134,7 @@ func (b Measurer) redundancyOf(ctx context.Context, path string) *health.Redunda
switch {
case fstype == "zfs":
pool, _, _ := strings.Cut(source, "/")
r := &health.Redundancy{Kind: "zfs", Where: pool}
r := &diskreadings.Redundancy{Kind: "zfs", Where: pool}
health, err := b.Run(ctx, "zpool", "list", "-H", "-o", "health", pool)
if err != nil {
r.Said = "zpool list: " + err.Error()
@@ -156,7 +156,7 @@ func (b Measurer) redundancyOf(ctx context.Context, path string) *health.Redunda
return r
case strings.HasPrefix(source, "/dev/md"):
device := strings.TrimPrefix(source, "/dev/")
r := &health.Redundancy{Kind: "md", Where: device}
r := &diskreadings.Redundancy{Kind: "md", Where: device}
mdstat, err := b.Run(ctx, "cat", "/proc/mdstat")
if err != nil {
r.Said = err.Error()
@@ -175,7 +175,7 @@ func (b Measurer) redundancyOf(ctx context.Context, path string) *health.Redunda
}
return r
case fstype == "btrfs":
r := &health.Redundancy{Kind: "btrfs", Where: source}
r := &diskreadings.Redundancy{Kind: "btrfs", Where: source}
stats, err := b.Run(ctx, "btrfs", "device", "stats", "--check", path)
if err != nil {
r.Healthy, r.Said = &no, "device errors: "+oneLineOf(err.Error())
modules/disk-load/cmd/disk-load/packages_test.go
+32
View File
@@ -0,0 +1,32 @@
package main
import (
"encoding/json"
"os"
"testing"
)
// The readings' layout is disk-load's own package, published by its series for the readers to take (ADR 0634 §4).
func TestTheReadingsLayoutIsAPublishedPackageOfThisModule(t *testing.T) {
raw, err := os.ReadFile("../../module.json")
if err != nil {
t.Fatal(err)
}
var m struct {
Build struct {
Artifacts []struct {
Name, Kind, Series string
Paths []string
} `json:"artifacts"`
} `json:"build"`
}
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatal(err)
}
for _, a := range m.Build.Artifacts {
if a.Name == "diskreadings" && a.Kind == "package" && a.Series == "0.1" && len(a.Paths) == 1 && a.Paths[0] == "diskreadings" {
return
}
}
t.Errorf("no package artifact diskreadings of series 0.1 in %s", raw)
}
modules/disk-load/cmd/disk-load/readings.go
+3 -3
View File
@@ -1,12 +1,12 @@
package main
// The readings layout, their key and the peak are mesh-sdk's (health.Reading, health.NextReading; novox/hq ADR 0571
// The readings layout, their key and the peak are the package diskreadings (Reading, NextReading; novox/hq ADR 0571
// clause 3): disk-load measures each placed data item of its node and keeps the readings as its own state.
import "git.novox.be/novox/mesh-sdk/go/health"
import "git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings"
// StateReadings is the state disk-load keeps.
const StateReadings = health.ReadingsState
const StateReadings = diskreadings.ReadingsState
// StateRounds is its state of each node's last round, key `<node>` (novox/hq ADR 0577 clause 3): apart from the readings,
// so a reader of the readings never meets a round.
modules/disk-load/cmd/disk-load/readings_test.go
+4 -4
View File
@@ -9,8 +9,8 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/health"
)
type memStore struct {
@@ -84,11 +84,11 @@ func TestARoundMeasuresTheDeclaredItemsAndWritesWhatChanged(t *testing.T) {
if _, _, err := k.Round(context.Background(), false); err != nil {
t.Fatal(err)
}
if kept, _ := health.DecodeReading(store.m["ace.pg.data"]); !kept.RetiredAt.IsZero() {
if kept, _ := diskreadings.DecodeReading(store.m["ace.pg.data"]); !kept.RetiredAt.IsZero() {
t.Fatalf("a round retired a reading: %+v", kept)
}
k.MeasureModule(context.Background(), "pg")
gone, _ := health.DecodeReading(store.m["ace.pg.data"])
gone, _ := diskreadings.DecodeReading(store.m["ace.pg.data"])
if gone.RetiredAt.IsZero() || !strings.Contains(gone.RetiredWhy, "no longer declared on ace") || gone.FirstSeen.IsZero() {
t.Fatalf("a reading of an item no longer declared: %+v", gone)
}
@@ -96,7 +96,7 @@ func TestARoundMeasuresTheDeclaredItemsAndWritesWhatChanged(t *testing.T) {
if _, _, err := k.Round(context.Background(), false); err != nil {
t.Fatal(err)
}
if back, _ := health.DecodeReading(store.m["ace.pg.data"]); !back.RetiredAt.IsZero() {
if back, _ := diskreadings.DecodeReading(store.m["ace.pg.data"]); !back.RetiredAt.IsZero() {
t.Fatalf("a reading declared again is still retired: %+v", back)
}
if _, _, err := k.MeasureNow(context.Background(), "pg", "tmp"); err == nil {
modules/disk-load/cmd/disk-load/readings_tools.go
+10 -10
View File
@@ -8,8 +8,8 @@ import (
"strings"
"time"
"git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/health"
)
// ReadingTools are the tools over the state `readings`: every key, one key, how full the state is, and one item
@@ -35,7 +35,7 @@ func ReadingTools(k *Keeper) []stdio.Tool {
if len(raw) == 0 {
return nil, fmt.Errorf("no reading is kept at %q", key)
}
r, err := health.DecodeReading(raw)
r, err := diskreadings.DecodeReading(raw)
return r, err
}
keys, err := k.Store.Keys()
@@ -45,12 +45,12 @@ func ReadingTools(k *Keeper) []stdio.Tool {
sort.Strings(keys)
module := text(args, "module")
type line struct {
Key string `json:"key"`
SizeBytes *int64 `json:"size_bytes,omitempty"`
PeakBytes *int64 `json:"peak_bytes,omitempty"`
Precision string `json:"precision,omitempty"`
Error string `json:"error,omitempty"`
Reading health.Reading `json:"-"`
Key string `json:"key"`
SizeBytes *int64 `json:"size_bytes,omitempty"`
PeakBytes *int64 `json:"peak_bytes,omitempty"`
Precision string `json:"precision,omitempty"`
Error string `json:"error,omitempty"`
Reading diskreadings.Reading `json:"-"`
}
var lines []line
total := 0
@@ -60,7 +60,7 @@ func ReadingTools(k *Keeper) []stdio.Tool {
return nil, err
}
total += len(raw)
r, err := health.DecodeReading(raw)
r, err := diskreadings.DecodeReading(raw)
if err != nil || (module != "" && r.Module != module) {
continue
}
@@ -118,7 +118,7 @@ func ReadingTools(k *Keeper) []stdio.Tool {
if err != nil {
return nil, err
}
return map[string]any{"written": written, "reading": r, "key": health.ReadingKey(k.Node, module, item),
return map[string]any{"written": written, "reading": r, "key": diskreadings.ReadingKey(k.Node, module, item),
"note": strings.TrimSpace(map[bool]string{true: "the reading changed and was written", false: "the reading had not changed"}[written])}, nil
},
},
modules/disk-load/cmd/disk-load/whole_integration_test.go
+5 -5
View File
@@ -26,9 +26,9 @@ import (
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/testbus"
)
@@ -195,7 +195,7 @@ func TestDiskLoadWholeMeasuresAnItemWithinAMinuteOfItsKey(t *testing.T) {
if err != nil {
t.Fatal(err)
}
for _, b := range []string{delivery.EngineDataBucket, health.ReadingsBucket, "disk-load_" + StateRounds} {
for _, b := range []string{delivery.EngineDataBucket, diskreadings.ReadingsBucket, "disk-load_" + StateRounds} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil {
t.Fatal(err)
}
@@ -240,11 +240,11 @@ func TestDiskLoadWholeMeasuresAnItemWithinAMinuteOfItsKey(t *testing.T) {
if _, err := data.Put(ctx, key, raw); err != nil {
t.Fatal(err)
}
readings, _ := js.KeyValue(ctx, health.ReadingsBucket)
readings, _ := js.KeyValue(ctx, diskreadings.ReadingsBucket)
for {
e, err := readings.Get(ctx, health.ReadingKey("ace", "pg", "data"))
e, err := readings.Get(ctx, diskreadings.ReadingKey("ace", "pg", "data"))
if err == nil {
r, err := health.DecodeReading(e.Value())
r, err := diskreadings.DecodeReading(e.Value())
if err != nil || r.Path != dir || r.MeasuredAt == nil {
t.Fatalf("the reading is %s (%v)", e.Value(), err)
}