novox/mesh-catalog
18
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Readings and last good nights are the owners' published packages: disk-load's diskreadings and restic's backupgood, taken by series (B5b Left 3; ADR 0634 §4; issue 1966)

Browse Source 
Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i
This commit is contained in:
jschoubben
2026-11-05 14:48:10 +01:00
parent c7ae96414f
commit e0eecbf3c8
43 changed files with 565 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/disk-load/cmd/disk-load/data_test.go
+4 -4
View File
@@ -9,10 +9,10 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
)
// ADR 0577 clauses 2, 3 (and the disk-load check of clause 5): the items measured come from the node-engine's keys of
@@ -137,7 +137,7 @@ func TestTheItemsComeFromTheEnginesKeysAndARoundSaysHowFarItGot(t *testing.T) {
// A restart: oldie's key was deleted while the watch was down, so no delete arrives. Once the watch handed over its
// keys, oldie's reading is retired and pg's kept; an empty set retires nothing (the engine has not written yet).
_ = readings.Put("ace.oldie.x", health.Reading{V: health.ReadingVersion, Node: "ace",
_ = readings.Put("ace.oldie.x", diskreadings.Reading{V: diskreadings.ReadingVersion, Node: "ace",
PlacedDataItem: catalogue.PlacedDataItem{Module: "oldie", ID: "x", Path: "/srv/oldie"}})
for _, restart := range []struct {
name string
@@ -167,12 +167,12 @@ func TestTheItemsComeFromTheEnginesKeysAndARoundSaysHowFarItGot(t *testing.T) {
}
}
func readingOf(s *memStore, key string) *health.Reading {
func readingOf(s *memStore, key string) *diskreadings.Reading {
raw, _ := s.Get(key)
if raw == nil {
return nil
}
r, err := health.DecodeReading(raw)
r, err := diskreadings.DecodeReading(raw)
if err != nil {
return nil
}
modules/disk-load/cmd/disk-load/keeper.go
+18 -18
View File
@@ -6,9 +6,9 @@ import (
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
"os"
"os/exec"
"sort"
@@ -287,7 +287,7 @@ func (k *Keeper) MeasureModule(ctx context.Context, module string) {
// asked only on a change of that module's key (novox/hq ADR 0577 clause 2), never from a round. It is cleared when the
// item is declared again (measureOne).
func (k *Keeper) retireGone(module string, declared []catalogue.PlacedDataItem) (int, error) {
return k.retire(func(r health.Reading) bool { return r.Module == module }, declared)
return k.retire(func(r diskreadings.Reading) bool { return r.Module == module }, declared)
}
// RetireUnkeyed retires, once the watch handed over every current key, the readings of this node whose module has no
@@ -310,7 +310,7 @@ func (k *Keeper) RetireUnkeyed(ctx context.Context) {
for _, its := range items {
declared = append(declared, its...)
}
n, err := k.retire(func(r health.Reading) bool { _, keyed := items[r.Module]; return !keyed }, declared)
n, err := k.retire(func(r diskreadings.Reading) bool { _, keyed := items[r.Module]; return !keyed }, declared)
switch {
case err != nil:
k.Say("the readings of modules with no key are not all retired: %v", err)
@@ -320,14 +320,14 @@ func (k *Keeper) RetireUnkeyed(ctx context.Context) {
}
// retire marks retired every reading of this node that of picks and declared does not name.
func (k *Keeper) retire(of func(health.Reading) bool, declared []catalogue.PlacedDataItem) (int, error) {
func (k *Keeper) retire(of func(diskreadings.Reading) bool, declared []catalogue.PlacedDataItem) (int, error) {
keys, err := k.Store.Keys()
if err != nil {
return 0, err
}
declaredNow := map[string]bool{}
for _, it := range declared {
declaredNow[health.ReadingKey(k.Node, it.Module, it.ID)] = true
declaredNow[diskreadings.ReadingKey(k.Node, it.Module, it.ID)] = true
}
now := k.Measurer.Now().UTC()
n := 0
@@ -339,7 +339,7 @@ func (k *Keeper) retire(of func(health.Reading) bool, declared []catalogue.Place
if err != nil || len(raw) == 0 {
continue
}
r, err := health.DecodeReading(raw)
r, err := diskreadings.DecodeReading(raw)
if err != nil || r.Node != k.Node || !of(r) || !r.RetiredAt.IsZero() {
continue
}
@@ -357,14 +357,14 @@ func (k *Keeper) retire(of func(health.Reading) bool, declared []catalogue.Place
// when it changed. It answers whether it wrote the reading, what the measurement said failed ("" when it did not, or
// when the item was not measured now), and an error when the reading could not be read or kept.
func (k *Keeper) measureOne(ctx context.Context, it catalogue.PlacedDataItem, walks bool) (bool, string, error) {
key := health.ReadingKey(k.Node, it.Module, it.ID)
var prev *health.Reading
key := diskreadings.ReadingKey(k.Node, it.Module, it.ID)
var prev *diskreadings.Reading
raw, err := k.Store.Get(key)
if err != nil {
return false, "", err
}
if len(raw) > 0 {
if p, err := health.DecodeReading(raw); err == nil {
if p, err := diskreadings.DecodeReading(raw); err == nil {
prev = &p
}
}
@@ -374,7 +374,7 @@ func (k *Keeper) measureOne(ctx context.Context, it catalogue.PlacedDataItem, wa
return false, "", nil
}
m := k.Measurer.Measure(ctx, it)
next, changed := health.NextReading(prev, k.Node, it, m, now)
next, changed := diskreadings.NextReading(prev, k.Node, it, m, now)
if prev != nil && !prev.RetiredAt.IsZero() {
// Declared again: it is no longer retired.
next.RetiredAt, next.RetiredWhy, changed = time.Time{}, "", true
@@ -386,32 +386,32 @@ func (k *Keeper) measureOne(ctx context.Context, it catalogue.PlacedDataItem, wa
}
// MeasureNow measures one item now, walk or not, and writes its reading when it changed.
func (k *Keeper) MeasureNow(ctx context.Context, module, item string) (health.Reading, bool, error) {
func (k *Keeper) MeasureNow(ctx context.Context, module, item string) (diskreadings.Reading, bool, error) {
k.mu.Lock()
defer k.mu.Unlock()
items, err := k.Items(ctx)
if err != nil {
return health.Reading{}, false, err
return diskreadings.Reading{}, false, err
}
for _, it := range items[module] {
if it.ID != item {
continue
}
if it.Class == ClassCache {
return health.Reading{}, false, fmt.Errorf("%s.%s is a cache, which is listed and never measured", module, item)
return diskreadings.Reading{}, false, fmt.Errorf("%s.%s is a cache, which is listed and never measured", module, item)
}
w, _, err := k.measureOne(ctx, it, true)
if err != nil {
return health.Reading{}, false, err
return diskreadings.Reading{}, false, err
}
raw, err := k.Store.Get(health.ReadingKey(k.Node, module, item))
raw, err := k.Store.Get(diskreadings.ReadingKey(k.Node, module, item))
if err != nil {
return health.Reading{}, w, err
return diskreadings.Reading{}, w, err
}
r, err := health.DecodeReading(raw)
r, err := diskreadings.DecodeReading(raw)
return r, w, err
}
return health.Reading{}, false, fmt.Errorf("%s declares no data item %q on this machine", module, item)
return diskreadings.Reading{}, false, fmt.Errorf("%s declares no data item %q on this machine", module, item)
}
// Loop runs a round after a few minutes and then every `every`, with the walks of the night's round when it is the
modules/disk-load/cmd/disk-load/measure.go
+7 -7
View File
@@ -6,8 +6,8 @@ package main
import (
"context"
"fmt"
"git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/health"
"regexp"
"strconv"
"strings"
@@ -56,9 +56,9 @@ func (b Measurer) exists(ctx context.Context, path string) bool {
}
// Measure is one item, measured now, by its own method.
func (b Measurer) Measure(ctx context.Context, it catalogue.PlacedDataItem) health.Measured {
func (b Measurer) Measure(ctx context.Context, it catalogue.PlacedDataItem) diskreadings.Measured {
at := b.Now().UTC()
m := health.Measured{MeasuredAt: &at}
m := diskreadings.Measured{MeasuredAt: &at}
if !b.exists(ctx, it.Path) {
m.Error = it.Path + " does not exist"
zero := int64(0)
@@ -120,7 +120,7 @@ func (b Measurer) Measure(ctx context.Context, it catalogue.PlacedDataItem) heal
// redundancyOf is the redundant storage a path is on: a ZFS pool's health, an md array's members, a btrfs
// filesystem's error counters. Nil for storage with no redundancy it knows how to read.
func (b Measurer) redundancyOf(ctx context.Context, path string) *health.Redundancy {
func (b Measurer) redundancyOf(ctx context.Context, path string) *diskreadings.Redundancy {
out, err := b.Run(ctx, "findmnt", "-no", "SOURCE,FSTYPE", "--target", path)
if err != nil {
return nil
@@ -134,7 +134,7 @@ func (b Measurer) redundancyOf(ctx context.Context, path string) *health.Redunda
switch {
case fstype == "zfs":
pool, _, _ := strings.Cut(source, "/")
r := &health.Redundancy{Kind: "zfs", Where: pool}
r := &diskreadings.Redundancy{Kind: "zfs", Where: pool}
health, err := b.Run(ctx, "zpool", "list", "-H", "-o", "health", pool)
if err != nil {
r.Said = "zpool list: " + err.Error()
@@ -156,7 +156,7 @@ func (b Measurer) redundancyOf(ctx context.Context, path string) *health.Redunda
return r
case strings.HasPrefix(source, "/dev/md"):
device := strings.TrimPrefix(source, "/dev/")
r := &health.Redundancy{Kind: "md", Where: device}
r := &diskreadings.Redundancy{Kind: "md", Where: device}
mdstat, err := b.Run(ctx, "cat", "/proc/mdstat")
if err != nil {
r.Said = err.Error()
@@ -175,7 +175,7 @@ func (b Measurer) redundancyOf(ctx context.Context, path string) *health.Redunda
}
return r
case fstype == "btrfs":
r := &health.Redundancy{Kind: "btrfs", Where: source}
r := &diskreadings.Redundancy{Kind: "btrfs", Where: source}
stats, err := b.Run(ctx, "btrfs", "device", "stats", "--check", path)
if err != nil {
r.Healthy, r.Said = &no, "device errors: "+oneLineOf(err.Error())
modules/disk-load/cmd/disk-load/packages_test.go
+32
View File
@@ -0,0 +1,32 @@
package main
import (
"encoding/json"
"os"
"testing"
)
// The readings' layout is disk-load's own package, published by its series for the readers to take (ADR 0634 §4).
func TestTheReadingsLayoutIsAPublishedPackageOfThisModule(t *testing.T) {
raw, err := os.ReadFile("../../module.json")
if err != nil {
t.Fatal(err)
}
var m struct {
Build struct {
Artifacts []struct {
Name, Kind, Series string
Paths []string
} `json:"artifacts"`
} `json:"build"`
}
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatal(err)
}
for _, a := range m.Build.Artifacts {
if a.Name == "diskreadings" && a.Kind == "package" && a.Series == "0.1" && len(a.Paths) == 1 && a.Paths[0] == "diskreadings" {
return
}
}
t.Errorf("no package artifact diskreadings of series 0.1 in %s", raw)
}
modules/disk-load/cmd/disk-load/readings.go
+3 -3
View File
@@ -1,12 +1,12 @@
package main
// The readings layout, their key and the peak are mesh-sdk's (health.Reading, health.NextReading; novox/hq ADR 0571
// The readings layout, their key and the peak are the package diskreadings (Reading, NextReading; novox/hq ADR 0571
// clause 3): disk-load measures each placed data item of its node and keeps the readings as its own state.
import "git.novox.be/novox/mesh-sdk/go/health"
import "git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings"
// StateReadings is the state disk-load keeps.
const StateReadings = health.ReadingsState
const StateReadings = diskreadings.ReadingsState
// StateRounds is its state of each node's last round, key `<node>` (novox/hq ADR 0577 clause 3): apart from the readings,
// so a reader of the readings never meets a round.
modules/disk-load/cmd/disk-load/readings_test.go
+4 -4
View File
@@ -9,8 +9,8 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/health"
)
type memStore struct {
@@ -84,11 +84,11 @@ func TestARoundMeasuresTheDeclaredItemsAndWritesWhatChanged(t *testing.T) {
if _, _, err := k.Round(context.Background(), false); err != nil {
t.Fatal(err)
}
if kept, _ := health.DecodeReading(store.m["ace.pg.data"]); !kept.RetiredAt.IsZero() {
if kept, _ := diskreadings.DecodeReading(store.m["ace.pg.data"]); !kept.RetiredAt.IsZero() {
t.Fatalf("a round retired a reading: %+v", kept)
}
k.MeasureModule(context.Background(), "pg")
gone, _ := health.DecodeReading(store.m["ace.pg.data"])
gone, _ := diskreadings.DecodeReading(store.m["ace.pg.data"])
if gone.RetiredAt.IsZero() || !strings.Contains(gone.RetiredWhy, "no longer declared on ace") || gone.FirstSeen.IsZero() {
t.Fatalf("a reading of an item no longer declared: %+v", gone)
}
@@ -96,7 +96,7 @@ func TestARoundMeasuresTheDeclaredItemsAndWritesWhatChanged(t *testing.T) {
if _, _, err := k.Round(context.Background(), false); err != nil {
t.Fatal(err)
}
if back, _ := health.DecodeReading(store.m["ace.pg.data"]); !back.RetiredAt.IsZero() {
if back, _ := diskreadings.DecodeReading(store.m["ace.pg.data"]); !back.RetiredAt.IsZero() {
t.Fatalf("a reading declared again is still retired: %+v", back)
}
if _, _, err := k.MeasureNow(context.Background(), "pg", "tmp"); err == nil {
modules/disk-load/cmd/disk-load/readings_tools.go
+10 -10
View File
@@ -8,8 +8,8 @@ import (
"strings"
"time"
"git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/health"
)
// ReadingTools are the tools over the state `readings`: every key, one key, how full the state is, and one item
@@ -35,7 +35,7 @@ func ReadingTools(k *Keeper) []stdio.Tool {
if len(raw) == 0 {
return nil, fmt.Errorf("no reading is kept at %q", key)
}
r, err := health.DecodeReading(raw)
r, err := diskreadings.DecodeReading(raw)
return r, err
}
keys, err := k.Store.Keys()
@@ -45,12 +45,12 @@ func ReadingTools(k *Keeper) []stdio.Tool {
sort.Strings(keys)
module := text(args, "module")
type line struct {
Key string `json:"key"`
SizeBytes *int64 `json:"size_bytes,omitempty"`
PeakBytes *int64 `json:"peak_bytes,omitempty"`
Precision string `json:"precision,omitempty"`
Error string `json:"error,omitempty"`
Reading health.Reading `json:"-"`
Key string `json:"key"`
SizeBytes *int64 `json:"size_bytes,omitempty"`
PeakBytes *int64 `json:"peak_bytes,omitempty"`
Precision string `json:"precision,omitempty"`
Error string `json:"error,omitempty"`
Reading diskreadings.Reading `json:"-"`
}
var lines []line
total := 0
@@ -60,7 +60,7 @@ func ReadingTools(k *Keeper) []stdio.Tool {
return nil, err
}
total += len(raw)
r, err := health.DecodeReading(raw)
r, err := diskreadings.DecodeReading(raw)
if err != nil || (module != "" && r.Module != module) {
continue
}
@@ -118,7 +118,7 @@ func ReadingTools(k *Keeper) []stdio.Tool {
if err != nil {
return nil, err
}
return map[string]any{"written": written, "reading": r, "key": health.ReadingKey(k.Node, module, item),
return map[string]any{"written": written, "reading": r, "key": diskreadings.ReadingKey(k.Node, module, item),
"note": strings.TrimSpace(map[bool]string{true: "the reading changed and was written", false: "the reading had not changed"}[written])}, nil
},
},
modules/disk-load/cmd/disk-load/whole_integration_test.go
+5 -5
View File
@@ -26,9 +26,9 @@ import (
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/testbus"
)
@@ -195,7 +195,7 @@ func TestDiskLoadWholeMeasuresAnItemWithinAMinuteOfItsKey(t *testing.T) {
if err != nil {
t.Fatal(err)
}
for _, b := range []string{delivery.EngineDataBucket, health.ReadingsBucket, "disk-load_" + StateRounds} {
for _, b := range []string{delivery.EngineDataBucket, diskreadings.ReadingsBucket, "disk-load_" + StateRounds} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil {
t.Fatal(err)
}
@@ -240,11 +240,11 @@ func TestDiskLoadWholeMeasuresAnItemWithinAMinuteOfItsKey(t *testing.T) {
if _, err := data.Put(ctx, key, raw); err != nil {
t.Fatal(err)
}
readings, _ := js.KeyValue(ctx, health.ReadingsBucket)
readings, _ := js.KeyValue(ctx, diskreadings.ReadingsBucket)
for {
e, err := readings.Get(ctx, health.ReadingKey("ace", "pg", "data"))
e, err := readings.Get(ctx, diskreadings.ReadingKey("ace", "pg", "data"))
if err == nil {
r, err := health.DecodeReading(e.Value())
r, err := diskreadings.DecodeReading(e.Value())
if err != nil || r.Path != dir || r.MeasuredAt == nil {
t.Fatalf("the reading is %s (%v)", e.Value(), err)
}
modules/disk-load/diskreadings/go.mod
+5
View File
@@ -0,0 +1,5 @@
module git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings
go 1.26.0
require git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000
modules/disk-load/diskreadings/readings.go
+137
View File
@@ -0,0 +1,137 @@
package diskreadings
import (
"encoding/json"
"fmt"
"reflect"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// **Disk usage readings and peaks per data item** (novox/hq ADR 0571 clause 3, ADR 0233 §21, §27, §34): disk-load, on every
// node, measures each data item the modules of its node declare (catalogue.PlacedDataItem, from compose/frombus.DataItems)
// and keeps the newest reading and the item's largest size in seven days as its own state `readings`, one key
// `<node>.<module>.<item>`, written only when it changes. The node-backup holder, mounts, nfs-server and the self-check's
// probes read it there; the controller keeps none of it. The package is disk-load's own, published by its series (ADR 0634 §4); the readers import it.
const (
DiskLoadModule = "disk-load"
ReadingsState = "readings"
ReadingsBucket = DiskLoadModule + "_" + ReadingsState
ReadingsRead = DiskLoadModule + "." + ReadingsState
)
// ReadingVersion is the version of a reading's layout.
const ReadingVersion = 1
// PeakWindow is how far back the peak looks (ADR 0233 §34).
const PeakWindow = 7 * 24 * time.Hour
// Redundancy is the redundant storage an item is on, as read: zfs, md or btrfs, its pool, array or filesystem, and its
// health (nil when it could not be read).
type Redundancy struct {
Kind string `json:"kind"`
Where string `json:"where"`
Healthy *bool `json:"healthy"`
Said string `json:"said"`
}
// Measured is what one measurement found. Precision says what the size is: "exact", "dataset <name>: its whole size",
// "partial: ..." (a lower bound, never compared), or "no size: ...".
type Measured struct {
SizeBytes *int64 `json:"size_bytes,omitempty"`
LastWrite *time.Time `json:"last_write,omitempty"`
MeasuredAt *time.Time `json:"measured_at,omitempty"`
Precision string `json:"precision,omitempty"`
Error string `json:"error,omitempty"`
Redundancy *Redundancy `json:"redundancy,omitempty"`
}
// Partial says the size is only a lower bound, which a peak is never made of.
func (m Measured) Partial() bool { return strings.HasPrefix(m.Precision, "partial") }
// DaySize is the largest size measured on one UTC day.
type DaySize struct {
Day string `json:"day"`
Bytes int64 `json:"bytes"`
}
// Reading is the value of one key: the item as placed, what was measured, and the largest size in the seven days up to
// it (Peak, measured on PeakAt), read from the day maxima Days.
type Reading struct {
V int `json:"v"`
Node string `json:"node"`
catalogue.PlacedDataItem
Measured
Peak *int64 `json:"peak_bytes,omitempty"`
PeakAt string `json:"peak_day,omitempty"`
Days []DaySize `json:"days,omitempty"`
// FirstSeen is when the item was first measured on this node, kept by every reading after; RetiredAt and RetiredWhy
// say its module left the node, written by whoever retires it and carried on unchanged. Each is left out while zero,
// so a reading written before reads as it did.
FirstSeen time.Time `json:"first_seen,omitzero"`
RetiredAt time.Time `json:"retired_at,omitzero"`
RetiredWhy string `json:"retired_why,omitempty"`
}
// ReadingKey is the key of an item on a node: `<node>.<module>.<item>`.
func ReadingKey(node, module, item string) string { return node + "." + module + "." + item }
// DecodeReading reads a key's value; it refuses another version.
func DecodeReading(raw []byte) (Reading, error) {
var r Reading
if err := json.Unmarshal(raw, &r); err != nil {
return Reading{}, fmt.Errorf("diskreadings: a reading is not readable: %w", err)
}
if r.V != ReadingVersion {
return Reading{}, fmt.Errorf("diskreadings: a reading of version %d, and this reads %d", r.V, ReadingVersion)
}
return r, nil
}
// NextReading is the reading after a measurement, and whether it differs from the one before in anything but the time it
// was taken: a reading that did not change is not written. A measurement with no size (an error, a shallow item) carries
// the last size's days on, so the peak outlives one failed measurement.
func NextReading(prev *Reading, node string, it catalogue.PlacedDataItem, m Measured, now time.Time) (Reading, bool) {
r := Reading{V: ReadingVersion, Node: node, PlacedDataItem: it, Measured: m, FirstSeen: now.UTC()}
var days []DaySize
if prev != nil {
days = prev.Days
r.RetiredAt, r.RetiredWhy = prev.RetiredAt, prev.RetiredWhy
if !prev.FirstSeen.IsZero() {
r.FirstSeen = prev.FirstSeen
}
}
today := now.UTC().Format("2006-01-02")
cut := now.UTC().Add(-PeakWindow).Format("2006-01-02")
var kept []DaySize
todays := int64(-1)
for _, d := range days {
switch {
case d.Day == today:
todays = d.Bytes
case d.Day >= cut:
kept = append(kept, d)
}
}
if m.SizeBytes != nil && !m.Partial() && *m.SizeBytes > todays {
todays = *m.SizeBytes
}
if todays >= 0 {
kept = append(kept, DaySize{Day: today, Bytes: todays})
}
r.Days = kept
for _, d := range kept {
if r.Peak == nil || d.Bytes >= *r.Peak {
b := d.Bytes
r.Peak, r.PeakAt = &b, d.Day
}
}
if prev == nil {
return r, true
}
a, b := *prev, r
a.MeasuredAt, b.MeasuredAt = nil, nil
return r, !reflect.DeepEqual(a, b)
}
modules/disk-load/diskreadings/readings_test.go
+55
View File
@@ -0,0 +1,55 @@
package diskreadings
import (
"encoding/json"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// ADR 0571 clause 3: a reading keeps its seven-day peak and is written only when it changes; a partial size is no peak.
func TestAReadingKeepsItsPeakAndIsWrittenOnlyWhenItChanges(t *testing.T) {
at := time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC)
var prev *Reading
for i, step := range []struct {
day, size int64
partial, wrote bool
peak int64
}{{20, 100, false, true, 100}, {20, 100, false, false, 100}, {21, 300, false, true, 300}, {22, 50, false, true, 300},
{23, 9000, true, true, 300}, {29, 10, false, true, 50}, {30, 10, false, true, 10}} {
now := time.Date(2026, 10, int(step.day), 1, 0, 0, 0, time.UTC)
size := step.size
m := Measured{SizeBytes: &size, MeasuredAt: &now, Precision: "exact"}
if step.partial {
m.Precision = "partial: stopped"
}
r, wrote := NextReading(prev, "ace", catalogue.PlacedDataItem{Module: "pg", ID: "data", Path: "/srv/pg"}, m, now)
raw, _ := json.Marshal(r)
back, err := DecodeReading(raw)
if wrote != step.wrote || r.Peak == nil || *r.Peak != step.peak || err != nil || back.ID != "data" {
t.Fatalf("reading %d: wrote %v, peak %v, read back %v", i, wrote, r.Peak, err)
}
prev = &r
}
// The first reading sets FirstSeen and every one after keeps it, as it keeps a retirement; an older reading reads as before.
if first := time.Date(2026, 10, 20, 1, 0, 0, 0, time.UTC); !prev.FirstSeen.Equal(first) {
t.Fatalf("first seen %s, not %s", prev.FirstSeen, first)
}
retired := *prev
retired.RetiredAt, retired.RetiredWhy = at, "its module left ace"
if r, _ := NextReading(&retired, "ace", retired.PlacedDataItem, retired.Measured, at.Add(time.Hour)); !r.RetiredAt.Equal(at) ||
r.RetiredWhy != "its module left ace" || !r.FirstSeen.Equal(prev.FirstSeen) {
t.Fatalf("a retired reading moved on as %+v", r)
}
if old, err := DecodeReading([]byte(`{"v":1,"node":"ace","module":"pg","item":"data"}`)); err != nil || !old.FirstSeen.IsZero() {
t.Fatalf("a reading of before read as %+v: %v", old, err)
}
if raw, _ := json.Marshal(Reading{V: ReadingVersion}); strings.Contains(string(raw), "first_seen") || strings.Contains(string(raw), "retired") {
t.Fatalf("zero fields were written: %s", raw)
}
if ReadingKey("ace", "pg", "data") != "ace.pg.data" || ReadingsBucket != "disk-load_readings" {
t.Fatal("the readings' key or bucket moved")
}
}
modules/disk-load/go.mod
+4
View File
@@ -21,3 +21,7 @@ require (
golang.org/x/sys v0.48.0 // indirect
golang.org/x/time v0.16.0 // indirect
)
require git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings => ./diskreadings
modules/disk-load/module.json
+9
View File
@@ -24,6 +24,15 @@
"loads": [
"disk-load"
]
},
{
"name": "diskreadings",
"kind": "package",
"language": "go",
"paths": [
"diskreadings"
],
"series": "0.1"
}
]
},
modules/mesh-self-check/cmd/mesh-self-check/manifest_test.go
+10 -3
View File
@@ -9,9 +9,10 @@ import (
)
type manifest struct {
Module string `json:"module"`
RunsAs string `json:"runs-as"`
Claims []struct {
Packages map[string]string `json:"packages"`
Module string `json:"module"`
RunsAs string `json:"runs-as"`
Claims []struct {
Name string `json:"name"`
Scope string `json:"scope"`
Serves []string `json:"serves"`
@@ -43,6 +44,12 @@ func readManifest(t *testing.T) manifest {
// itself a tool that starts it by hand (ADR 0564 §12).
func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) {
m := readManifest(t)
// The readings' layouts are the owners' published packages, taken by their series (ADR 0634 §4).
for _, pk := range []string{"diskreadings", "backupgood"} {
if m.Packages[pk] != "0.1" {
t.Errorf("packages %v: want %s of series 0.1", m.Packages, pk)
}
}
if m.Module != Module || m.RunsAs != Module {
t.Errorf("module %q runs as %q", m.Module, m.RunsAs)
}
modules/mesh-self-check/cmd/mesh-self-check/probes_data.go
+15 -13
View File
@@ -19,6 +19,8 @@ import (
"sync"
"time"
"git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings"
"git.novox.be/novox/mesh-catalog/modules/restic/backupgood"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
@@ -26,8 +28,8 @@ import (
// The states D13 reads (declared under `reads`).
const (
ReadingsRead = health.ReadingsRead
GoodRead = health.GoodRead
ReadingsRead = diskreadings.ReadingsRead
GoodRead = backupgood.GoodRead
ModulesRead = delivery.IndexerModule + ".modules"
ManifestsRead = delivery.IndexerModule + ".manifests"
)
@@ -52,8 +54,8 @@ const (
// dataFacts is everything D13 judges: the readings, the last good nights, each module's manifest, and the machines whose
// backup holder was heard from.
type dataFacts struct {
readings []health.Reading
good map[string]time.Time // by health.GoodKey
readings []diskreadings.Reading
good map[string]time.Time // by backupgood.GoodKey
manifests map[string]catalogue.Manifest
// unmeasured are the machines that hold node-backup, are heard from, and have no reading of anything.
unmeasured []string
@@ -139,7 +141,7 @@ func (c *Checker) dataFacts(ctx context.Context) (dataFacts, error) {
}
byNode := map[string]int{}
for _, k := range slices.Sorted(maps.Keys(states[ReadingsRead])) {
r, err := health.DecodeReading(states[ReadingsRead][k])
r, err := diskreadings.DecodeReading(states[ReadingsRead][k])
if err != nil {
continue
}
@@ -147,8 +149,8 @@ func (c *Checker) dataFacts(ctx context.Context) (dataFacts, error) {
byNode[r.Node]++
}
for _, raw := range states[GoodRead] {
if g, derr := health.DecodeGoodRow(raw); derr == nil && !g.LastGood.IsZero() {
f.good[health.GoodKey(g.Node, g.Module, g.Item)] = g.LastGood
if g, derr := backupgood.DecodeGoodRow(raw); derr == nil && !g.LastGood.IsZero() {
f.good[backupgood.GoodKey(g.Node, g.Module, g.Item)] = g.LastGood
}
}
for name, raw := range states[ModulesRead] {
@@ -247,7 +249,7 @@ func sizeWords(size *int64) string {
// firstSeen is when an item was first measured on its node (the reading keeps it); for a reading written before it did,
// the oldest day its seven-day peak is made of, else its measurement: a lower bound on how long it has been declared.
func firstSeen(r health.Reading, now time.Time) time.Time {
func firstSeen(r diskreadings.Reading, now time.Time) time.Time {
if !r.FirstSeen.IsZero() {
return r.FirstSeen
}
@@ -270,22 +272,22 @@ func obs(kind string, id, machine string, sev health.Severity, summary, said str
// dataFindings is every condition the data on record raises now: a function of what is known, as the controller's was.
func dataFindings(f dataFacts) []Observation {
var out []Observation
arrays := map[string][]health.Reading{}
arrays := map[string][]diskreadings.Reading{}
type shrunk struct {
machine, dataset, class string
size, peak int64
items []string
}
shrunkDatasets := map[string]shrunk{}
byItem := map[string][]health.Reading{}
retiredCopies := map[string][]health.Reading{}
byItem := map[string][]diskreadings.Reading{}
retiredCopies := map[string][]diskreadings.Reading{}
for _, m := range f.unmeasured {
o := obs(kindDataUnmeasured, m, m, Warning, fmt.Sprintf("%s's backup holder did not say what it measured of the data "+
"declared there, so nothing about that data is known", m), "disk-load keeps no reading of this machine's data")
o.Confirm = true
out = append(out, o)
}
rs := append([]health.Reading(nil), f.readings...)
rs := append([]diskreadings.Reading(nil), f.readings...)
sort.Slice(rs, func(i, j int) bool {
return keyOfItem(rs[i].Node, rs[i].Module, rs[i].ID) < keyOfItem(rs[j].Node, rs[j].Module, rs[j].ID)
})
@@ -354,7 +356,7 @@ func dataFindings(f dataFacts) []Observation {
// plan, said only where the machine was measured — where a holder is there to take it.
if item.BackedUp() && (class == catalogue.ClassIrreplaceable || r.MeasuredAt != nil) {
within := item.BackupWithin()
last, has := f.good[health.GoodKey(r.Node, r.Module, r.ID)]
last, has := f.good[backupgood.GoodKey(r.Node, r.Module, r.ID)]
switch {
case !has && f.now.Sub(firstSeen(r, f.now)) > within:
out = append(out, obs(kindBackupStale, id, r.Node, severity,
modules/mesh-self-check/cmd/mesh-self-check/probes_test.go
+14 -10
View File
@@ -9,6 +9,8 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings"
"git.novox.be/novox/mesh-catalog/modules/restic/backupgood"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
@@ -173,8 +175,8 @@ func TestTheBusProbes(t *testing.T) {
})},
{"D13 no good backup is counted from when the item was first measured", probeData, bus{},
[]string{"machine.ace.pg.data.backup-stale"}, "", dataMesh(func(r *dataRig) {
delete(r.good, health.ReadingKey("ace", "pg", "data"))
delete(r.good, health.ReadingKey("ace", "plex", "meta"))
delete(r.good, diskreadings.ReadingKey("ace", "pg", "data"))
delete(r.good, diskreadings.ReadingKey("ace", "plex", "meta"))
r.firstSeen("ace", "pg", "data", -100*time.Hour).firstSeen("ace", "plex", "meta", -time.Hour)
})},
{"D13 a machine with a backup holder and no reading", probeData, bus{}, []string{"machine.g14.data-unmeasured"}, "",
@@ -361,18 +363,20 @@ var dataNow = time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC)
// dataRig is the mesh D13 reads: a manifest per module with the items it declares, a reading per item, a last good night.
type dataRig struct {
s *memStore
readings map[string]*health.Reading
readings map[string]*diskreadings.Reading
good map[string]time.Time
}
func (r *dataRig) key(node, module, item string) string { return health.ReadingKey(node, module, item) }
func (r *dataRig) key(node, module, item string) string {
return diskreadings.ReadingKey(node, module, item)
}
func (r *dataRig) reading(node, module, item string) *health.Reading {
func (r *dataRig) reading(node, module, item string) *diskreadings.Reading {
k := r.key(node, module, item)
if r.readings[k] == nil {
at := dataNow
r.readings[k] = &health.Reading{V: health.ReadingVersion, Node: node, PlacedDataItem: catalogue.PlacedDataItem{Module: module,
ID: item, Class: catalogue.ClassValuable, Path: "/srv/" + module + "/" + item}, Measured: health.Measured{MeasuredAt: &at, Precision: "exact"}}
r.readings[k] = &diskreadings.Reading{V: diskreadings.ReadingVersion, Node: node, PlacedDataItem: catalogue.PlacedDataItem{Module: module,
ID: item, Class: catalogue.ClassValuable, Path: "/srv/" + module + "/" + item}, Measured: diskreadings.Measured{MeasuredAt: &at, Precision: "exact"}}
}
return r.readings[k]
}
@@ -400,7 +404,7 @@ func (r *dataRig) dataset(node, module, item, name string) *dataRig {
}
func (r *dataRig) array(node, module, item, kind, where string, healthy bool) *dataRig {
r.reading(node, module, item).Redundancy = &health.Redundancy{Kind: kind, Where: where, Healthy: &healthy, Said: "a member is missing"}
r.reading(node, module, item).Redundancy = &diskreadings.Redundancy{Kind: kind, Where: where, Healthy: &healthy, Said: "a member is missing"}
return r
}
@@ -436,7 +440,7 @@ func (r *dataRig) holder(node string) {
// declares movies (protected by redundancy), thumbs, meta and library (all valuable); every item is read as held, and what
// is given changes it.
func dataMesh(change func(*dataRig)) *memStore {
r := &dataRig{s: newStore(), readings: map[string]*health.Reading{}, good: map[string]time.Time{}}
r := &dataRig{s: newStore(), readings: map[string]*diskreadings.Reading{}, good: map[string]time.Time{}}
r.holder("ace")
declare := func(module string, items map[string]catalogue.DataItem) {
var man catalogue.Manifest
@@ -472,7 +476,7 @@ func dataMesh(change func(*dataRig)) *memStore {
for k, t := range r.good {
node, rest, _ := strings.Cut(k, ".")
module, item, _ := strings.Cut(rest, ".")
_ = r.s.Put(GoodRead, k, health.GoodRow{V: health.GoodVersion, Node: node, Module: module, Item: item, CoveredBy: "/x", LastGood: t})
_ = r.s.Put(GoodRead, k, backupgood.GoodRow{V: backupgood.GoodVersion, Node: node, Module: module, Item: item, CoveredBy: "/x", LastGood: t})
}
return r.s
}
modules/mesh-self-check/go.mod
+8
View File
@@ -23,3 +23,11 @@ require (
golang.org/x/sys v0.48.0 // indirect
golang.org/x/time v0.16.0 // indirect
)
require git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings => ../disk-load/diskreadings
require git.novox.be/novox/mesh-catalog/modules/restic/backupgood v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-catalog/modules/restic/backupgood => ../restic/backupgood
modules/mesh-self-check/module.json
+4
View File
@@ -1,6 +1,10 @@
{
"module": "mesh-self-check",
"version": "1",
"packages": {
"backupgood": "0.1",
"diskreadings": "0.1"
},
"runs-as": "mesh-self-check",
"claims": [
{
modules/mounts/cmd/mounts/bus.go
+2 -2
View File
@@ -14,7 +14,7 @@ package main
import (
"encoding/json"
"fmt"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings"
"os"
"path/filepath"
"time"
@@ -124,7 +124,7 @@ func (m *Machine) leave(p string, v any) error {
type Asker func(key string, body any) (json.RawMessage, error)
// Readings reads disk-load's readings of one machine's data items (its state `readings`, ADR 0571 clause 3).
type Readings func(node string) ([]health.Reading, error)
type Readings func(node string) ([]diskreadings.Reading, error)
// LeaveData reads disk-load for where the mesh keeps data on this machine and leaves the paths for the
// process (finding 4d). A failed ask keeps the last paths, with why; with none, the process sets up
modules/mounts/cmd/mounts/data.go
+2 -2
View File
@@ -8,7 +8,7 @@ package main
import (
"errors"
"fmt"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings"
"io"
"os"
"path/filepath"
@@ -65,7 +65,7 @@ func sortWants(w []Want) {
// PathsFrom is the paths disk-load's readings name: where the mesh keeps data here. No reading is an error, never an
// empty list: an empty list would let every mount point through.
func PathsFrom(readings []health.Reading) ([]string, error) {
func PathsFrom(readings []diskreadings.Reading) ([]string, error) {
if len(readings) == 0 {
return nil, errors.New("disk-load keeps no reading of this machine's data yet")
}
modules/mounts/cmd/mounts/main.go
+5 -5
View File
@@ -13,7 +13,7 @@ import (
"context"
"encoding/json"
"fmt"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings"
"os"
"os/signal"
"strings"
@@ -64,13 +64,13 @@ func shareOnTheBus(m *Machine) {
_, err := stdio.State(SharesState).Put(key, value)
return err
}
read := func(node string) ([]health.Reading, error) {
st := stdio.State(health.ReadingsRead)
read := func(node string) ([]diskreadings.Reading, error) {
st := stdio.State(diskreadings.ReadingsRead)
keys, err := st.Keys()
if err != nil {
return nil, err
}
var out []health.Reading
var out []diskreadings.Reading
for _, k := range keys {
if !strings.HasPrefix(k, node+".") {
continue
@@ -82,7 +82,7 @@ func shareOnTheBus(m *Machine) {
if e == nil {
continue
}
if r, err := health.DecodeReading(e.Value); err == nil {
if r, err := diskreadings.DecodeReading(e.Value); err == nil {
out = append(out, r)
}
}
modules/mounts/cmd/mounts/manifest_test.go
+7
View File
@@ -10,6 +10,7 @@ import (
)
type manifest struct {
Packages map[string]string `json:"packages"`
Tools []string `json:"tools"`
Requires []string `json:"requires"`
Reads []string `json:"reads"`
@@ -46,6 +47,12 @@ func readManifest(t *testing.T) (manifest, string) {
// or acts; the module requires the share, asks the server's seat and holds node-mounts.
func TestTheManifestNamesWhatTheBundleServesAndWhatItNeeds(t *testing.T) {
m, raw := readManifest(t)
// The readings' layouts are the owners' published packages, taken by their series (ADR 0634 §4).
for _, pk := range []string{"diskreadings"} {
if m.Packages[pk] != "0.1" {
t.Errorf("packages %v: want %s of series 0.1", m.Packages, pk)
}
}
var tools, verbs []string
for _, tl := range Tools(newSystem(t).machine()) {
if !strings.HasSuffix(tl.Description, "(r)") && !strings.HasSuffix(tl.Description, "(a)") {
modules/mounts/cmd/mounts/review_test.go
+5 -5
View File
@@ -5,8 +5,8 @@ package main
import (
"context"
"encoding/json"
"git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/health"
"os"
"path/filepath"
"strings"
@@ -200,7 +200,7 @@ func TestWithoutTheMeshsDataNothingNewIsSetUp(t *testing.T) {
func TestTheBundleLeavesTheMeshsDataForTheProcess(t *testing.T) {
f := newSystem(t)
f.write(ConfigFile, libraryOnly)
rs := []health.Reading{{V: 1, Node: "workstation", PlacedDataItem: catalogue.PlacedDataItem{Module: "plex", ID: "data", Path: "/mnt/plex/data"}}}
rs := []diskreadings.Reading{{V: 1, Node: "workstation", PlacedDataItem: catalogue.PlacedDataItem{Module: "plex", ID: "data", Path: "/mnt/plex/data"}}}
paths, err := PathsFrom(rs)
if err != nil || len(paths) != 1 || paths[0] != "/mnt/plex/data" {
t.Fatalf("paths %q %v", paths, err)
@@ -284,13 +284,13 @@ func callVerb(t *testing.T, m *Machine, name string, args map[string]any) (any,
func TestTheBundleKeepsTheLastDataWhenAnAskFails(t *testing.T) {
f := newSystem(t)
m := f.machine()
ok := func(node string) ([]health.Reading, error) {
return []health.Reading{{V: 1, Node: node, PlacedDataItem: catalogue.PlacedDataItem{Module: "plex", ID: "data", Path: "/mnt/plex/data"}}}, nil
ok := func(node string) ([]diskreadings.Reading, error) {
return []diskreadings.Reading{{V: 1, Node: node, PlacedDataItem: catalogue.PlacedDataItem{Module: "plex", ID: "data", Path: "/mnt/plex/data"}}}, nil
}
if said := m.LeaveData("workstation", ok, time.Now()); len(said) != 0 {
t.Fatalf("said %q", said)
}
failing := func(string) ([]health.Reading, error) { return nil, os.ErrDeadlineExceeded }
failing := func(string) ([]diskreadings.Reading, error) { return nil, os.ErrDeadlineExceeded }
said := m.LeaveData("workstation", failing, time.Now())
d, err := readJSON[DataPaths](m, DataFile)
if err != nil || len(d.Paths) != 1 || d.Paths[0] != "/mnt/plex/data" || d.Error == "" || len(said) == 0 {
modules/mounts/go.mod
+4
View File
@@ -13,3 +13,7 @@ require (
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/sys v0.48.0 // indirect
)
require git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings => ../disk-load/diskreadings
modules/mounts/module.json
+3
View File
@@ -1,6 +1,9 @@
{
"module": "mounts",
"version": "1",
"packages": {
"diskreadings": "0.1"
},
"requires": [
"nfs-share"
],
modules/nfs-server/cmd/nfs-server/backup.go
+4 -4
View File
@@ -18,8 +18,8 @@ package main
import (
"encoding/json"
"fmt"
"git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
"os"
"path"
"sort"
@@ -50,7 +50,7 @@ type BackupFromBus struct {
// (novox/hq ADR 0571 clause 4), and the paths that module keeps data in on that machine from disk-load's readings
// (clause 3). A machine whose backup seat nobody holds, and no readings at all, are errors: never an empty list read as
// "nothing to fear".
func BackupFrom(machine string, keys map[string][]byte, readings []health.Reading) (string, []string, error) {
func BackupFrom(machine string, keys map[string][]byte, readings []diskreadings.Reading) (string, []string, error) {
holders, err := delivery.HoldersOf(keys, BackupSeat, machine)
if err != nil {
return "", nil, err
@@ -84,7 +84,7 @@ func (m *Machine) TakeBackup(machine string) {
// TakeBackupWith is TakeBackup with the ask and the reading of the seat-holders state given, for tests. A failed ask keeps the last holder and
// paths and adds why.
func (m *Machine) TakeBackupWith(machine string, readings func(node string) ([]health.Reading, error),
func (m *Machine) TakeBackupWith(machine string, readings func(node string) ([]diskreadings.Reading, error),
holderKeys func() (map[string][]byte, error)) {
var last BackupFromBus
if text, err := m.Read(BackupFile); err == nil {
@@ -94,7 +94,7 @@ func (m *Machine) TakeBackupWith(machine string, readings func(node string) ([]h
next.Looked = m.Now().UTC().Format(time.RFC3339)
next.Error = ""
seats, err := holderKeys()
var data []health.Reading
var data []diskreadings.Reading
if err == nil {
data, err = readings(machine)
}
modules/nfs-server/cmd/nfs-server/backup_test.go
+8 -8
View File
@@ -4,8 +4,8 @@ import (
"context"
"encoding/json"
"errors"
"git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/health"
"strings"
"testing"
"time"
@@ -101,11 +101,11 @@ func TestBackupFromTheSeatsAndTheData(t *testing.T) {
func TestAFailedAskKeepsTheLastBackupPaths(t *testing.T) {
f := newFake(t)
m := f.machine()
m.TakeBackupWith("ace", func(string) ([]health.Reading, error) {
m.TakeBackupWith("ace", func(string) ([]diskreadings.Reading, error) {
return nil, errors.New("no responders")
}, noHolders)
f.backup("restic", "/storage/restic")
m.TakeBackupWith("ace", func(string) ([]health.Reading, error) {
m.TakeBackupWith("ace", func(string) ([]diskreadings.Reading, error) {
return nil, errors.New("timeout")
}, func() (map[string][]byte, error) { return backupHolders("ace", "restic"), nil })
var b BackupFromBus
@@ -154,12 +154,12 @@ func TestAStaleOrUnknownBackupListIsSaidWithItsAge(t *testing.T) {
t.Errorf("an unknown backup list is not a warning in the health: %+v", h)
}
ok := func(string) ([]health.Reading, error) {
ok := func(string) ([]diskreadings.Reading, error) {
return readingsOfPaths("hs", map[string]string{"restic/repository": "/srv/restic"}), nil
}
m.TakeBackupWith("hs", ok, func() (map[string][]byte, error) { return backupHolders("hs", "restic"), nil })
now = now.Add(2 * time.Hour)
m.TakeBackupWith("hs", func(string) ([]health.Reading, error) { return nil, errors.New("the bus did not answer") },
m.TakeBackupWith("hs", func(string) ([]diskreadings.Reading, error) { return nil, errors.New("the bus did not answer") },
func() (map[string][]byte, error) { return backupHolders("hs", "restic"), nil })
s := m.Pass(context.Background(), nil)
said := strings.Join(append(append([]string{}, s.Said...), s.Warnings...), "\n")
@@ -181,11 +181,11 @@ func backupHolders(node, module string) map[string][]byte {
func noHolders() (map[string][]byte, error) { return nil, errors.New("the state is not read yet") }
// readingsOfPaths is disk-load's readings of one node's items, by "<module>/<item>", each at its path.
func readingsOfPaths(node string, items map[string]string) []health.Reading {
var out []health.Reading
func readingsOfPaths(node string, items map[string]string) []diskreadings.Reading {
var out []diskreadings.Reading
for k, p := range items {
module, item, _ := strings.Cut(k, "/")
out = append(out, health.Reading{V: 1, Node: node, PlacedDataItem: catalogue.PlacedDataItem{Module: module, ID: item, Path: p}})
out = append(out, diskreadings.Reading{V: 1, Node: node, PlacedDataItem: catalogue.PlacedDataItem{Module: module, ID: item, Path: p}})
}
return out
}
modules/nfs-server/cmd/nfs-server/guards_test.go
+2 -2
View File
@@ -4,7 +4,7 @@ package main
import (
"context"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings"
"strings"
"testing"
)
@@ -65,7 +65,7 @@ func TestAFailedReloadIsTriedAgainWhenNothingElseAsks(t *testing.T) {
func TestBackupFromRefusesNoReadingsAtAll(t *testing.T) {
seats := backupHolders("ace", "restic")
if _, paths, err := BackupFrom("ace", seats, []health.Reading{}); err == nil {
if _, paths, err := BackupFrom("ace", seats, []diskreadings.Reading{}); err == nil {
t.Errorf("no readings were read as the empty list %v: never \"nothing to fear\"", paths)
}
}
modules/nfs-server/cmd/nfs-server/main.go
+5 -5
View File
@@ -20,11 +20,11 @@ import (
"syscall"
"time"
"git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/views"
)
@@ -98,13 +98,13 @@ func ShareExports(m *Machine) {
}
// readingsOf reads disk-load's readings of one machine's data items, every key `<node>.<module>.<item>`.
func readingsOf(node string) ([]health.Reading, error) {
state := stdio.State(health.ReadingsRead)
func readingsOf(node string) ([]diskreadings.Reading, error) {
state := stdio.State(diskreadings.ReadingsRead)
names, err := state.Keys()
if err != nil {
return nil, err
}
var out []health.Reading
var out []diskreadings.Reading
for _, k := range names {
if !strings.HasPrefix(k, node+".") {
continue
@@ -116,7 +116,7 @@ func readingsOf(node string) ([]health.Reading, error) {
if e == nil {
continue
}
if r, err := health.DecodeReading(e.Value); err == nil {
if r, err := diskreadings.DecodeReading(e.Value); err == nil {
out = append(out, r)
}
}
modules/nfs-server/cmd/nfs-server/manifest_test.go
+11 -4
View File
@@ -2,7 +2,7 @@ package main
import (
"encoding/json"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings"
"os"
"regexp"
"sort"
@@ -11,8 +11,9 @@ import (
)
type manifest struct {
Tools []string `json:"tools"`
Claims []struct {
Packages map[string]string `json:"packages"`
Tools []string `json:"tools"`
Claims []struct {
Name string `json:"name"`
Scope string `json:"scope"`
Serves []string `json:"serves"`
@@ -44,6 +45,12 @@ func readManifest(t *testing.T) (manifest, string) {
// whether it reads or acts.
func TestTheSeatsVerbsAreAllServedAndTheHealthToolListed(t *testing.T) {
m, _ := readManifest(t)
// The readings' layouts are the owners' published packages, taken by their series (ADR 0634 §4).
for _, pk := range []string{"diskreadings"} {
if m.Packages[pk] != "0.1" {
t.Errorf("packages %v: want %s of series 0.1", m.Packages, pk)
}
}
if len(m.Claims) != 1 || m.Claims[0].Name != Seat || m.Claims[0].Scope != "node" {
t.Fatalf("claims: %+v", m.Claims)
}
@@ -141,7 +148,7 @@ func TestTheManifestCarriesGrantsTheRosterTheWantsAndTheRemovalOrder(t *testing.
t.Fatal(err)
}
// The assigner's seat-holders is asked once and followed by its events (ADR 0615), no longer read.
if len(full.Reads) != 2 || full.Reads[0] != health.ReadingsRead || full.Reads[1] != MountsState {
if len(full.Reads) != 2 || full.Reads[0] != diskreadings.ReadingsRead || full.Reads[1] != MountsState {
t.Errorf("reads: %v", full.Reads)
}
if len(full.Invokes) != 2 || full.Invokes[0] != SeatHoldersTool || full.Invokes[1] != "seat:mesh-hand-acts.hand-act" || len(full.Consumes) != 1 || full.Consumes[0] != "mesh-delivery-assigner.views.seat-holders" {
modules/nfs-server/go.mod
+4
View File
@@ -15,3 +15,7 @@ require (
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/sys v0.48.0 // indirect
)
require git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings => ../disk-load/diskreadings
modules/nfs-server/module.json
+3
View File
@@ -1,6 +1,9 @@
{
"module": "nfs-server",
"version": "1",
"packages": {
"diskreadings": "0.1"
},
"upgrade": {
"policy": "record",
"why": "the folders other machines mount: a build that breaks the exports leaves every client's mount hanging or refused, and the gate on this one machine does not see the clients (hq ADR 0236, ADR 0263)"
modules/restic/backupgood/go.mod
+3
View File
@@ -0,0 +1,3 @@
module git.novox.be/novox/mesh-catalog/modules/restic/backupgood
go 1.26.0
modules/restic/backupgood/good.go
+45
View File
@@ -0,0 +1,45 @@
package backupgood
import (
"encoding/json"
"fmt"
"time"
)
// **Each item's last good night** (novox/hq ADR 0571 clause 3, ADR 0564 §6; the lead, 2026-10-29): the node-backup holder
// restic keeps, from its tools bundle through the runtime on the night's file event (its night's process runs as the
// operator's account and holds no bus account, ADR 0436 §1-2), its own state `good`, one key `<node>.<module>.<item>` for every item a module
// declares that a backup path covers, written when it changes: the item, the path whose snapshot keeps it and the newest
// good night of its module (zero before the first). The self-check's probe of a stale backup reads it there. The package is restic's own, published by its series (ADR 0634 §4); the readers import it.
const (
ResticModule = "restic"
GoodState = "good"
GoodBucket = ResticModule + "_" + GoodState
GoodRead = ResticModule + "." + GoodState
GoodVersion = 1
)
// GoodRow is one key of restic's `good`.
type GoodRow struct {
V int `json:"v"`
Node string `json:"node"`
Module string `json:"module"`
Item string `json:"item"`
CoveredBy string `json:"covered_by"`
LastGood time.Time `json:"last_good,omitzero"`
}
// GoodKey is a row's key: `<node>.<module>.<item>`, as a disk-load reading's.
func GoodKey(node, module, item string) string { return node + "." + module + "." + item }
// DecodeGoodRow reads a row; it refuses another version.
func DecodeGoodRow(raw []byte) (GoodRow, error) {
var g GoodRow
if err := json.Unmarshal(raw, &g); err != nil {
return GoodRow{}, fmt.Errorf("backupgood: a last good night is not readable: %w", err)
}
if g.V != GoodVersion {
return GoodRow{}, fmt.Errorf("backupgood: a last good night of version %d, and this reads %d", g.V, GoodVersion)
}
return g, nil
}
modules/restic/backupgood/good_test.go
+20
View File
@@ -0,0 +1,20 @@
package backupgood
import (
"encoding/json"
"testing"
"time"
)
// Restic's last good night per item, keyed as a disk-load reading is.
func TestALastGoodNightIsReadBackAndAnotherVersionRefused(t *testing.T) {
at := time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC)
goodRow, _ := json.Marshal(GoodRow{V: GoodVersion, Node: "ace", Module: "pg", Item: "data", CoveredBy: "/srv/pg", LastGood: at})
if g, err := DecodeGoodRow(goodRow); err != nil || !g.LastGood.Equal(at) || GoodKey("ace", "pg", "data") != "ace.pg.data" ||
GoodBucket != "restic_good" || GoodRead != "restic.good" {
t.Fatalf("a last good night read back as %+v: %v", g, err)
}
if _, err := DecodeGoodRow([]byte(`{"v":2}`)); err == nil {
t.Fatal("a last good night of another version was read")
}
}
modules/restic/cmd/restic-backups/backups.go
+5 -4
View File
@@ -25,7 +25,8 @@ import (
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings"
"git.novox.be/novox/mesh-catalog/modules/restic/backupgood"
"os"
"os/exec"
"path/filepath"
@@ -209,10 +210,10 @@ type Backups struct {
Node string
// Readings is every reading disk-load keeps for this machine (its state `readings`, ADR 0571 clause 3); nil where the
// process has no runtime to read it.
Readings func() ([]health.Reading, error)
// PutGood writes one row of the state `good` (health.GoodRow): the night's process through its own bus account, the
Readings func() ([]diskreadings.Reading, error)
// PutGood writes one row of the state `good` (backupgood.GoodRow): the night's process through its own bus account, the
// tools through the runtime; nil where there is no bus to write to.
PutGood func(key string, row health.GoodRow) error
PutGood func(key string, row backupgood.GoodRow) error
goodMu sync.Mutex
goodWrote map[string][]byte
modules/restic/cmd/restic-backups/data.go
+6 -6
View File
@@ -15,8 +15,8 @@ import (
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/health"
"os"
"path/filepath"
"regexp"
@@ -32,7 +32,7 @@ type Item = catalogue.PlacedDataItem
// its newest good backup.
type ItemReport struct {
Item
health.Measured
diskreadings.Measured
// Peak is the largest size in the last seven days, on PeakDay.
Peak *int64 `json:"peak_bytes,omitempty"`
PeakDay string `json:"peak_day,omitempty"`
@@ -59,8 +59,8 @@ func (b *Backups) Items() (map[string][]Item, error) {
// Measurements is the newest reading disk-load keeps of each item on this machine, by module and item (its state
// `readings`, ADR 0571 clause 3); none where it cannot be read, which `backed-up` then says as no reading.
func (b *Backups) Measurements() map[string]map[string]health.Reading {
out := map[string]map[string]health.Reading{}
func (b *Backups) Measurements() map[string]map[string]diskreadings.Reading {
out := map[string]map[string]diskreadings.Reading{}
if b.Readings == nil {
return out
}
@@ -71,7 +71,7 @@ func (b *Backups) Measurements() map[string]map[string]health.Reading {
}
for _, r := range rs {
if out[r.Module] == nil {
out[r.Module] = map[string]health.Reading{}
out[r.Module] = map[string]diskreadings.Reading{}
}
out[r.Module][r.ID] = r
}
@@ -107,7 +107,7 @@ func (b *Backups) recordGood(module string, at time.Time) {
// itemReports is every item of one module, with its newest reading and its newest good backup: the
// module's newest good night, where that night keeps the path that covers the item.
func itemReports(its []Item, measured map[string]health.Reading, paths []string, good time.Time) []ItemReport {
func itemReports(its []Item, measured map[string]diskreadings.Reading, paths []string, good time.Time) []ItemReport {
out := []ItemReport{}
for _, it := range its {
r := ItemReport{Item: it}
modules/restic/cmd/restic-backups/data_test.go
+9 -8
View File
@@ -6,8 +6,9 @@ package main
import (
"context"
"errors"
"git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings"
"git.novox.be/novox/mesh-catalog/modules/restic/backupgood"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/health"
"os"
"path/filepath"
"strings"
@@ -53,9 +54,9 @@ func TestEveryItemIsSaidWithItsReadingAndItsLastGoodBackup(t *testing.T) {
night := time.Date(2026, 10, 6, 3, 0, 0, 0, time.UTC)
b := &Backups{Where: where, Now: func() time.Time { return night.Add(time.Hour) }, Say: quiet,
Run: func(context.Context, string, ...string) (string, error) { return "[]", nil },
Readings: func() ([]health.Reading, error) {
return []health.Reading{{V: 1, Node: "ace", PlacedDataItem: catalogue.PlacedDataItem{Module: "postgres", ID: "store"},
Measured: health.Measured{SizeBytes: &size, LastWrite: &write, MeasuredAt: &at, Precision: "exact"}, Peak: &peak, PeakAt: "2026-10-05"}}, nil
Readings: func() ([]diskreadings.Reading, error) {
return []diskreadings.Reading{{V: 1, Node: "ace", PlacedDataItem: catalogue.PlacedDataItem{Module: "postgres", ID: "store"},
Measured: diskreadings.Measured{SizeBytes: &size, LastWrite: &write, MeasuredAt: &at, Precision: "exact"}, Peak: &peak, PeakAt: "2026-10-05"}}, nil
}}
b.recordGood("postgres", night)
got, err := b.BackedUp(context.Background(), "")
@@ -80,13 +81,13 @@ func TestEveryItemIsSaidWithItsReadingAndItsLastGoodBackup(t *testing.T) {
}
// The same last good night is kept on the bus beside good.json, one row per item a backup path covers.
b.Node = "ace"
wrote := map[string]health.GoodRow{}
b.PutGood = func(k string, r health.GoodRow) error { wrote[k] = r; return nil }
wrote := map[string]backupgood.GoodRow{}
b.PutGood = func(k string, r backupgood.GoodRow) error { wrote[k] = r; return nil }
b.recordGood("postgres", night) // a good night writes the rows, once; the same night again writes nothing
n := len(wrote)
b.recordGood("postgres", night)
row, ok := wrote[health.GoodKey("ace", "postgres", "store")]
if len(wrote) != 1 || n != 1 || !ok || !row.LastGood.Equal(night) || row.CoveredBy != "/var/lib/mesh-store/dumps" || row.V != health.GoodVersion {
row, ok := wrote[backupgood.GoodKey("ace", "postgres", "store")]
if len(wrote) != 1 || n != 1 || !ok || !row.LastGood.Equal(night) || row.CoveredBy != "/var/lib/mesh-store/dumps" || row.V != backupgood.GoodVersion {
t.Fatalf("the good rows are %+v", wrote)
}
}
modules/restic/cmd/restic-backups/good.go
+6 -6
View File
@@ -1,7 +1,7 @@
package main
// **Each item's last good night, on the bus** (novox/hq ADR 0571 clause 3, ADR 0564 §6): beside good.json, the module's
// own state `good` (health.GoodRow), one key `<node>.<module>.<item>`, written when a night's files change, so the
// own state `good` (backupgood.GoodRow), one key `<node>.<module>.<item>`, written when a night's files change, so the
// self-check's probe of a stale backup reads it there.
import (
@@ -12,12 +12,12 @@ import (
"time"
"unsafe"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-catalog/modules/restic/backupgood"
)
// GoodRows is the rows of this machine (health.GoodRow): every item a module declares that a path of the module's backup
// GoodRows is the rows of this machine (backupgood.GoodRow): every item a module declares that a path of the module's backup
// covers, with the module's newest good night.
func (b *Backups) GoodRows() (map[string]health.GoodRow, error) {
func (b *Backups) GoodRows() (map[string]backupgood.GoodRow, error) {
items, err := b.Items()
if err != nil {
return nil, err
@@ -31,7 +31,7 @@ func (b *Backups) GoodRows() (map[string]health.GoodRow, error) {
paths[d.Module] = d.Paths
}
good := b.Good()
out := map[string]health.GoodRow{}
out := map[string]backupgood.GoodRow{}
for module, its := range items {
for _, it := range its {
if it.Class == "cache" || it.CoveredBy == "" {
@@ -44,7 +44,7 @@ func (b *Backups) GoodRows() (map[string]health.GoodRow, error) {
if !covered {
continue
}
out[health.GoodKey(b.Node, module, it.ID)] = health.GoodRow{V: health.GoodVersion, Node: b.Node, Module: module,
out[backupgood.GoodKey(b.Node, module, it.ID)] = backupgood.GoodRow{V: backupgood.GoodVersion, Node: b.Node, Module: module,
Item: it.ID, CoveredBy: it.CoveredBy, LastGood: good[module].UTC()}
}
}
modules/restic/cmd/restic-backups/main.go
+10 -9
View File
@@ -23,9 +23,10 @@ import (
"strings"
"time"
"git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings"
"git.novox.be/novox/mesh-catalog/modules/restic/backupgood"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/announce"
"git.novox.be/novox/mesh-sdk/go/health"
)
// Seat is the role this module holds.
@@ -57,12 +58,12 @@ func main() {
counting(b)
case "":
// Readable: the self-check reads it (its manifest's `reads`); checks/readable holds it.
if err := stdio.Keeps(announce.State{Name: health.GoodState, Readable: true}); err != nil {
if err := stdio.Keeps(announce.State{Name: backupgood.GoodState, Readable: true}); err != nil {
say("%v", err)
os.Exit(1)
}
b.PutGood = func(key string, row health.GoodRow) error {
_, err := stdio.State(health.GoodState).Put(key, row)
b.PutGood = func(key string, row backupgood.GoodRow) error {
_, err := stdio.State(backupgood.GoodState).Put(key, row)
return err
}
go watchGood(b)
@@ -197,14 +198,14 @@ func tools(b *Backups) []stdio.Tool {
}
// busReadings reads disk-load's readings of this machine from its state, every key `<node>.<module>.<item>`.
func busReadings(node string) func() ([]health.Reading, error) {
return func() ([]health.Reading, error) {
st := stdio.State(health.ReadingsRead)
func busReadings(node string) func() ([]diskreadings.Reading, error) {
return func() ([]diskreadings.Reading, error) {
st := stdio.State(diskreadings.ReadingsRead)
keys, err := st.Keys()
if err != nil {
return nil, err
}
var out []health.Reading
var out []diskreadings.Reading
for _, k := range keys {
if !strings.HasPrefix(k, node+".") {
continue
@@ -213,7 +214,7 @@ func busReadings(node string) func() ([]health.Reading, error) {
if err != nil || e == nil {
continue
}
if r, err := health.DecodeReading(e.Value); err == nil {
if r, err := diskreadings.DecodeReading(e.Value); err == nil {
out = append(out, r)
}
}
modules/restic/cmd/restic-backups/packages_test.go
+32
View File
@@ -0,0 +1,32 @@
package main
import (
"encoding/json"
"os"
"testing"
)
// The readings' layout is restic's own package, published by its series for the readers to take (ADR 0634 §4).
func TestTheReadingsLayoutIsAPublishedPackageOfThisModule(t *testing.T) {
raw, err := os.ReadFile("../../module.json")
if err != nil {
t.Fatal(err)
}
var m struct {
Build struct {
Artifacts []struct {
Name, Kind, Series string
Paths []string
} `json:"artifacts"`
} `json:"build"`
}
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatal(err)
}
for _, a := range m.Build.Artifacts {
if a.Name == "backupgood" && a.Kind == "package" && a.Series == "0.1" && len(a.Paths) == 1 && a.Paths[0] == "backupgood" {
return
}
}
t.Errorf("no package artifact backupgood of series 0.1 in %s", raw)
}
modules/restic/go.mod
+8
View File
@@ -13,3 +13,11 @@ require (
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/sys v0.48.0 // indirect
)
require git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings => ../disk-load/diskreadings
require git.novox.be/novox/mesh-catalog/modules/restic/backupgood v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-catalog/modules/restic/backupgood => ./backupgood
modules/restic/module.json
+12
View File
@@ -1,6 +1,9 @@
{
"module": "restic",
"version": "1",
"packages": {
"diskreadings": "0.1"
},
"moved": true,
"claims": [
{
@@ -193,6 +196,15 @@
"name": "files",
"kind": "archive",
"from": "files"
},
{
"name": "backupgood",
"kind": "package",
"language": "go",
"paths": [
"backupgood"
],
"series": "0.1"
}
]
},