novox/mesh-catalog
18
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

restic: read each item's reading and peak from disk-load's state and the item list from the sdk; the measuring leaves the backup holder (issue 1955, ADR 0571 clause 3)

Browse Source
This commit is contained in:
jschoubben
2026-10-29 23:46:56 +01:00
parent f149768f5d
commit 44a3f24456
8 changed files with 134 additions and 577 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/restic/cmd/restic-backups/backups.go
+4
View File
@@ -25,6 +25,7 @@ import (
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-sdk/go/health"
"os"
"os/exec"
"path/filepath"
@@ -188,6 +189,9 @@ type Backups struct {
// measuring run as processes of their own). Node is the machine's name, said in the events.
Emit func(event string, body any) error
Node string
// Readings is every reading disk-load keeps for this machine (its state `readings`, ADR 0571 clause 3); nil where the
// process has no runtime to read it.
Readings func() ([]health.Reading, error)
}
// The locks, files in this module's state. The night runs in a process of its own and `now`, `restore` and a
modules/restic/cmd/restic-backups/data.go
+39 -352
View File
@@ -1,26 +1,9 @@
// The data the modules on this machine declare, measured (novox/hq ADR 0233).
// The data the modules on this machine declare (novox/hq ADR 0233), and what the readings say of them.
//
// The mesh composes a second file beside the backup lines: every data item every module on the
// machine declares, one line each,
//
// item <id> <class> <path> <covered-by> <protection> <measure>
//
// where covered-by is the path whose snapshot keeps it (the item itself, or the directory its dump
// writes into), or `-` when it is not backed up; protection is backup, redundancy, both, or none; and
// measure is how the item is measured. This holder measures each item that is not a cache — its size,
// its newest write, and the redundant storage it is on and whether that is healthy (ZFS, md, btrfs) —
// and says it, with each module's last good backup and the precision of the measurement, in
// `backed-up`.
//
// **Never an unbounded walk of something large.** Three methods, the item's own choice:
//
// - `walk` (the default, for small items): every file summed and the newest change found, in one walk
// as root — at most once a day, and stopped after walkFor or walkFiles, said as "measured partially";
// - `dataset`: the ZFS dataset holding the path — its `used` from the filesystem's own counters — and
// the newest change among the path's top-level entries; hourly, and nothing is walked;
// - `shallow`: the newest change among the top-level entries only, and no size; hourly. The controller keeps the readings and says when an item shrinks, stops being written,
//
// goes without a backup, or is replaced by an empty copy of itself; this holder only measures.
// The mesh composes a file beside the backup lines: every data item every module on the machine declares, one line
// each, which mesh-sdk reads (catalogue.ParseDataItems). disk-load measures the items and keeps each reading and its
// seven-day peak as its own state (ADR 0571 clause 3); this holder reads them there and says each, with each module's
// last good backup, in `backed-up`.
//
// And it deletes, when a person has decided: an item the mesh retired — its module gone from this
// machine — is removed by `backup_delete_retired`, and only after a last restore point of it has been
@@ -32,104 +15,32 @@ import (
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/health"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"sync"
"time"
)
// Item is one data item a module declares.
type Item struct {
// Measure is walk, dataset or shallow.
Measure string `json:"measure,omitempty"`
Module string `json:"-"`
ID string `json:"item"`
Class string `json:"class"`
Path string `json:"path"`
CoveredBy string `json:"covered_by,omitempty"`
Protection string `json:"protection,omitempty"`
}
// Item is one data item a module declares, as placed on this machine (mesh-sdk's reading of the holder's data text).
type Item = catalogue.PlacedDataItem
// Redundancy is the redundant storage an item is on, as read here.
type Redundancy struct {
// Kind is zfs, md or btrfs.
Kind string `json:"kind"`
// Where is the pool, the array device or the filesystem.
Where string `json:"where"`
// Healthy is nil when its state could not be read.
Healthy *bool `json:"healthy"`
Said string `json:"said"`
}
// Measured is what one measurement found.
type Measured struct {
SizeBytes *int64 `json:"size_bytes,omitempty"`
LastWrite *time.Time `json:"last_write,omitempty"`
MeasuredAt *time.Time `json:"measured_at,omitempty"`
// Precision says what the size is: "exact", "dataset <name>: its whole size", "partial: …" (a lower
// bound, never compared), or "no size: …".
Precision string `json:"precision,omitempty"`
Error string `json:"error,omitempty"`
Redundancy *Redundancy `json:"redundancy,omitempty"`
}
// ItemReport is one item as `backed-up` says it.
// ItemReport is one item as `backed-up` says it: the item, its newest reading from disk-load (ADR 0571 clause 3) and
// its newest good backup.
type ItemReport struct {
Item
Measured
health.Measured
// Peak is the largest size in the last seven days, on PeakDay.
Peak *int64 `json:"peak_bytes,omitempty"`
PeakDay string `json:"peak_day,omitempty"`
LastBackup *time.Time `json:"last_backup,omitempty"`
}
// The classes the mesh declares; a cache is listed and never measured.
const classCache = "cache"
var safePath = regexp.MustCompile(`^/[^\s'"\\$` + "`" + `]*$`)
// parseItems reads the composed data file into each module's items. A line this holder does not read
// is refused, naming the module, as a backup line is.
func parseItems(text string) (map[string][]Item, error) {
out := map[string][]Item{}
module := ""
for _, raw := range strings.Split(text, "\n") {
line := strings.TrimSpace(raw)
if line == "" {
continue
}
if m := moduleHeader.FindStringSubmatch(line); m != nil {
module = m[1]
continue
}
if strings.HasPrefix(line, "#") || module == "" {
continue
}
f := strings.Fields(line)
if len(f) < 5 || len(f) > 7 || f[0] != "item" || !safePath.MatchString(f[3]) || (f[4] != "-" && !safePath.MatchString(f[4])) {
return nil, fmt.Errorf("%s declares a data line this holder does not read: %s", module, line)
}
it := Item{Module: module, ID: f[1], Class: f[2], Path: f[3]}
if len(f) >= 6 {
it.Protection = f[5]
}
it.Measure = measureWalk
if len(f) == 7 {
it.Measure = f[6]
}
switch it.Measure {
case measureWalk, measureDataset, measureShallow:
default:
return nil, fmt.Errorf("%s declares a data line this holder does not read: %s", module, line)
}
if f[4] != "-" {
it.CoveredBy = f[4]
}
out[module] = append(out[module], it)
}
return out, nil
}
// Items is what the modules on this machine declare; none when the mesh composed no data file — an
// older controller, which composes none.
func (b *Backups) Items() (map[string][]Item, error) {
@@ -143,257 +54,30 @@ func (b *Backups) Items() (map[string][]Item, error) {
if err != nil {
return nil, err
}
return parseItems(string(raw))
return catalogue.ParseDataItems(string(raw))
}
func (b *Backups) measuredFile() string { return filepath.Join(b.Where.State, "measured.json") }
// Measurements is the newest measurement of each item, by module and item.
func (b *Backups) Measurements() map[string]map[string]Measured {
out := map[string]map[string]Measured{}
if raw, err := os.ReadFile(b.measuredFile()); err == nil {
_ = json.Unmarshal(raw, &out)
// Measurements is the newest reading disk-load keeps of each item on this machine, by module and item (its state
// `readings`, ADR 0571 clause 3); none where it cannot be read, which `backed-up` then says as no reading.
func (b *Backups) Measurements() map[string]map[string]health.Reading {
out := map[string]map[string]health.Reading{}
if b.Readings == nil {
return out
}
rs, err := b.Readings()
if err != nil {
b.Say("disk-load's readings could not be read: %v", err)
return out
}
for _, r := range rs {
if out[r.Module] == nil {
out[r.Module] = map[string]health.Reading{}
}
out[r.Module][r.ID] = r
}
return out
}
// The ways an item is measured.
const (
measureWalk = "walk"
measureDataset = "dataset"
measureShallow = "shallow"
)
// The bounds on a walk, and how often one runs: a walk is for small items, and one that meets a large
// one stops and says so rather than loading the disks for hours.
var (
walkFor = 10 * time.Minute
walkFiles = 2_000_000
walkEvery = 23 * time.Hour
)
// walkCommand sums the files under a path and finds its newest change, in one walk, as root — bounded
// by time and by files. One line out: "<bytes> <epoch seconds> <files> <complete 1|0>".
func walkCommand(path string) string {
return fmt.Sprintf("timeout %d find '%s' -xdev -printf '%%y %%s %%T@\\n' 2>/dev/null | head -n %d | "+
"awk 'BEGIN{s=0;m=0;n=0} {n++; if ($1==\"f\") s+=$2; if ($3>m) m=$3} END {printf \"%%d %%.0f %%d\\n\", s, m, n}'; "+
"echo \"${PIPESTATUS[0]:-0}\"", int(walkFor.Seconds()), path, walkFiles)
}
// topCommand is the newest change among a path and its top-level entries, and how many there are:
// one directory read, bounded. "<epoch seconds> <entries>".
func topCommand(path string) string {
return "timeout 60 find '" + path + "' -maxdepth 1 -printf '%T@\\n' 2>/dev/null | " +
"awk 'BEGIN{m=0;n=0} {n++; if ($1>m) m=$1} END {printf \"%.0f %d\\n\", m, n-1}'"
}
func epochTime(s string) *time.Time {
epoch, err := strconv.ParseInt(s, 10, 64)
if err != nil || epoch <= 0 {
return nil
}
t := time.Unix(epoch, 0).UTC()
return &t
}
// measure is one item, measured now, by its own method.
func (b *Backups) measure(ctx context.Context, it Item) Measured {
at := b.Now().UTC()
m := Measured{MeasuredAt: &at}
if !b.exists(ctx, it.Path) {
m.Error = it.Path + " does not exist"
zero := int64(0)
m.SizeBytes, m.Precision = &zero, "exact"
return m
}
m.Redundancy = b.redundancyOf(ctx, it.Path)
switch it.Measure {
case measureDataset, measureShallow:
out, err := b.Run(ctx, "bash", "-c", topCommand(it.Path))
if f := strings.Fields(out); err == nil && len(f) == 2 {
m.LastWrite = epochTime(f[0])
} else if err != nil {
m.Error = err.Error()
}
m.Precision = "no size: the newest change among its top-level entries only"
if it.Measure == measureShallow {
return m
}
out, err = b.Run(ctx, "zfs", "list", "-Hp", "-o", "name,used", it.Path)
f := strings.Fields(out)
if err != nil || len(f) != 2 {
m.Precision = "no size: it is on no ZFS dataset to read one from; the newest change among its top-level entries only"
return m
}
used, err := strconv.ParseInt(f[1], 10, 64)
if err != nil {
return m
}
m.SizeBytes = &used
m.Precision = "dataset " + f[0] + ": its whole size, from the filesystem's counters; the newest change among the top-level entries"
return m
}
out, err := b.Run(ctx, "bash", "-c", walkCommand(it.Path))
if err != nil {
m.Error = err.Error()
return m
}
lines := strings.Split(strings.TrimSpace(out), "\n")
f := strings.Fields(lines[0])
if len(f) != 3 || len(lines) < 2 {
m.Error = "the measurement said " + strings.TrimSpace(out)
return m
}
size, err1 := strconv.ParseInt(f[0], 10, 64)
files, err2 := strconv.Atoi(f[2])
if err1 != nil || err2 != nil {
m.Error = "the measurement said " + strings.TrimSpace(out)
return m
}
m.SizeBytes, m.LastWrite = &size, epochTime(f[1])
m.Precision = "exact"
if status := strings.TrimSpace(lines[len(lines)-1]); status == "124" || files >= walkFiles {
m.Precision = fmt.Sprintf("partial: measured partially — stopped after %s or %d files; the size is a lower bound, "+
"and this item wants measure: dataset or shallow", walkFor, walkFiles)
}
return m
}
// redundancyOf is the redundant storage a path is on, read as root: a ZFS pool's health and its own
// verdict, an md array's members, a btrfs filesystem's error counters. Nil for storage with no
// redundancy this holder knows how to read — which, for an item said to be protected by redundancy, the
// controller says is no protection at all.
func (b *Backups) redundancyOf(ctx context.Context, path string) *Redundancy {
out, err := b.Run(ctx, "findmnt", "-no", "SOURCE,FSTYPE", "--target", path)
if err != nil {
return nil
}
f := strings.Fields(out)
if len(f) < 2 {
return nil
}
source, fstype := f[0], f[1]
yes, no := true, false
switch {
case fstype == "zfs":
pool, _, _ := strings.Cut(source, "/")
r := &Redundancy{Kind: "zfs", Where: pool}
health, err := b.Run(ctx, "zpool", "list", "-H", "-o", "health", pool)
if err != nil {
r.Said = "zpool list: " + err.Error()
return r
}
status, err := b.Run(ctx, "zpool", "status", "-x", pool)
if err != nil {
r.Said = "zpool status: " + err.Error()
return r
}
health, status = strings.TrimSpace(health), strings.TrimSpace(status)
r.Said = "health " + health + "; " + firstLineOf(status)
if health == "ONLINE" && strings.Contains(status, "is healthy") {
r.Healthy = &yes
} else {
r.Healthy = &no
r.Said = "health " + health + "; " + oneLineOf(status)
}
return r
case strings.HasPrefix(source, "/dev/md"):
device := strings.TrimPrefix(source, "/dev/")
r := &Redundancy{Kind: "md", Where: device}
mdstat, err := b.Run(ctx, "cat", "/proc/mdstat")
if err != nil {
r.Said = err.Error()
return r
}
healthy, said, found := mdHealth(mdstat, device)
if !found {
r.Said = device + " is not in /proc/mdstat"
return r
}
r.Said = said
if healthy {
r.Healthy = &yes
} else {
r.Healthy = &no
}
return r
case fstype == "btrfs":
r := &Redundancy{Kind: "btrfs", Where: source}
stats, err := b.Run(ctx, "btrfs", "device", "stats", "--check", path)
if err != nil {
r.Healthy, r.Said = &no, "device errors: "+oneLineOf(err.Error())
return r
}
r.Healthy, r.Said = &yes, firstLineOf(stats)
return r
}
return nil
}
var mdMembers = regexp.MustCompile(`\[([U_]+)\]`)
// mdHealth reads one array's block of /proc/mdstat: healthy when every member is up and it is not
// recovering.
func mdHealth(mdstat, device string) (bool, string, bool) {
lines := strings.Split(mdstat, "\n")
for i, l := range lines {
if !strings.HasPrefix(l, device+" ") {
continue
}
block := l
for j := i + 1; j < len(lines) && strings.HasPrefix(lines[j], " "); j++ {
block += " " + strings.TrimSpace(lines[j])
}
members := mdMembers.FindStringSubmatch(block)
healthy := members != nil && !strings.Contains(members[1], "_") && !strings.Contains(block, "recovery")
return healthy, oneLineOf(block), true
}
return false, "", false
}
func firstLineOf(s string) string {
line, _, _ := strings.Cut(strings.TrimSpace(s), "\n")
return line
}
func oneLineOf(s string) string { return strings.Join(strings.Fields(s), " ") }
// MeasureAll measures every item that is not a cache, one at a time, and keeps what it found. An item
// measured by a walk is walked only when walks is true or its last walk is older than walkEvery: the
// hourly round reads counters, and a walk runs at most once a day.
func (b *Backups) MeasureAll(ctx context.Context, walks bool) error {
// The hourly round runs in its own process and the walk after a night in the night's (novox/hq issue 454).
release, err := b.hold(measureLock)
if err != nil {
return err
}
defer release()
items, err := b.Items()
if err != nil {
return err
}
before := b.Measurements()
found := map[string]map[string]Measured{}
for module, its := range items {
for _, it := range its {
if it.Class == classCache {
continue
}
if found[module] == nil {
found[module] = map[string]Measured{}
}
if was, ok := before[module][it.ID]; ok && it.Measure == measureWalk && !walks && was.MeasuredAt != nil &&
b.Now().Sub(*was.MeasuredAt) < walkEvery {
found[module][it.ID] = was
continue
}
found[module][it.ID] = b.measure(ctx, it)
}
}
raw, _ := json.MarshalIndent(found, "", " ")
return os.WriteFile(b.measuredFile(), append(raw, '\n'), 0o600)
}
func (b *Backups) goodFile() string { return filepath.Join(b.Where.State, "good.json") }
// Good is each module's newest good night: what a night that failed since does not erase.
@@ -420,12 +104,15 @@ func (b *Backups) recordGood(module string, at time.Time) {
}
}
// itemReports is every item of one module, with its newest measurement and its newest good backup: the
// itemReports is every item of one module, with its newest reading and its newest good backup: the
// module's newest good night, where that night keeps the path that covers the item.
func itemReports(its []Item, measured map[string]Measured, paths []string, good time.Time) []ItemReport {
func itemReports(its []Item, measured map[string]health.Reading, paths []string, good time.Time) []ItemReport {
out := []ItemReport{}
for _, it := range its {
r := ItemReport{Item: it, Measured: measured[it.ID]}
r := ItemReport{Item: it}
if m, ok := measured[it.ID]; ok {
r.Measured, r.Peak, r.PeakDay = m.Measured, m.Peak, m.PeakAt
}
if it.CoveredBy != "" && !good.IsZero() {
for _, p := range paths {
if p == it.CoveredBy {
modules/restic/cmd/restic-backups/data_test.go
+22 -141
View File
@@ -6,7 +6,8 @@ package main
import (
"context"
"errors"
"fmt"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/health"
"os"
"path/filepath"
"strings"
@@ -28,53 +29,34 @@ func withData(t *testing.T, declared, data string) Where {
}
func TestTheComposedDataFileIsReadIntoEachModulesItems(t *testing.T) {
got, err := parseItems(composedData)
b := &Backups{Where: withData(t, composed, composedData), Now: time.Now, Say: quiet}
got, err := b.Items()
must(t, err)
if len(got["postgres"]) != 2 || got["postgres"][0].CoveredBy != "/var/lib/mesh-store/dumps" || got["postgres"][1].CoveredBy != "" ||
got["searxng"][0].Class != "cache" {
t.Fatalf("%+v", got)
}
for _, bad := range []string{"# pg\nitem a irreplaceable relative -\n", "# pg\nitem a irreplaceable /x\n", "# pg\nitem a b /x; rm -rf / -\n"} {
if _, err := parseItems(bad); err == nil || !strings.Contains(err.Error(), "pg declares a data line") {
t.Errorf("%q: %v", bad, err)
}
}
// An older controller composes no data file: nothing is measured, nothing fails.
b := &Backups{Where: placed(t, composed), Now: time.Now, Say: quiet}
// An older controller composes no data file: nothing is declared, nothing fails.
b = &Backups{Where: placed(t, composed), Now: time.Now, Say: quiet}
if items, err := b.Items(); err != nil || len(items) != 0 {
t.Fatalf("%v, %v", items, err)
}
}
// Every item but a cache is measured — its files' size and its newest change, in one walk as root — and
// `backed-up` says each with the newest good night of the module that keeps the path covering it.
func TestEveryItemButACacheIsMeasuredAndSaidWithItsLastGoodBackup(t *testing.T) {
// `backed-up` says every item with disk-load's newest reading and its peak, and the newest good night of the module that
// keeps the path covering it; an item with no reading yet is said with none.
func TestEveryItemIsSaidWithItsReadingAndItsLastGoodBackup(t *testing.T) {
where := withData(t, composed, composedData)
var mu sync.Mutex
var walked []string
run := func(_ context.Context, name string, args ...string) (string, error) {
mu.Lock()
defer mu.Unlock()
switch {
case name == "test":
return "", nil
case name == "bash" && strings.Contains(args[1], "find '"):
walked = append(walked, args[1])
if strings.Contains(args[1], "'/var/lib/mesh-store'") {
return "1073741824 1759744800 4000\n0\n", nil
}
return "5000 1759700000 3\n0\n", nil
case name == "restic":
return "[]", nil
}
return "", nil
}
size, peak := int64(1<<30), int64(2<<30)
at := time.Date(2026, 10, 6, 4, 0, 0, 0, time.UTC)
write := time.Unix(1759744800, 0).UTC()
night := time.Date(2026, 10, 6, 3, 0, 0, 0, time.UTC)
b := &Backups{Where: where, Run: run, Now: func() time.Time { return night.Add(time.Hour) }, Say: quiet}
must(t, b.MeasureAll(context.Background(), true))
if len(walked) != 2 {
t.Fatalf("walked %d paths, want the two that are not a cache: %v", len(walked), walked)
}
b := &Backups{Where: where, Now: func() time.Time { return night.Add(time.Hour) }, Say: quiet,
Run: func(context.Context, string, ...string) (string, error) { return "[]", nil },
Readings: func() ([]health.Reading, error) {
return []health.Reading{{V: 1, Node: "ace", PlacedDataItem: catalogue.PlacedDataItem{Module: "postgres", ID: "store"},
Measured: health.Measured{SizeBytes: &size, LastWrite: &write, MeasuredAt: &at, Precision: "exact"}, Peak: &peak, PeakAt: "2026-10-05"}}, nil
}}
b.recordGood("postgres", night)
got, err := b.BackedUp(context.Background(), "")
must(t, err)
@@ -89,12 +71,12 @@ func TestEveryItemButACacheIsMeasuredAndSaidWithItsLastGoodBackup(t *testing.T)
}
}
}
if store == nil || store.SizeBytes == nil || *store.SizeBytes != 1<<30 || store.LastWrite == nil ||
store.LastWrite.Unix() != 1759744800 || store.LastBackup == nil || !store.LastBackup.Equal(night) {
if store == nil || store.SizeBytes == nil || *store.SizeBytes != 1<<30 || store.Peak == nil || *store.Peak != 2<<30 ||
store.LastWrite == nil || store.LastWrite.Unix() != 1759744800 || store.LastBackup == nil || !store.LastBackup.Equal(night) {
t.Fatalf("the store is said as %+v", store)
}
if dumps == nil || dumps.LastBackup != nil {
t.Fatalf("an item not backed up is said backed up: %+v", dumps)
if dumps == nil || dumps.LastBackup != nil || dumps.SizeBytes != nil {
t.Fatalf("an item with no reading or no backup is said with one: %+v", dumps)
}
}
@@ -202,104 +184,3 @@ func TestARetiredItemIsDeletedOnlyAfterALastRestorePoint(t *testing.T) {
t.Fatalf("%+v, %v", again, err)
}
}
// The redundant storage under an item is read: a ZFS pool healthy, then degraded; an md array with a
// member missing; and plain storage, which is no redundancy at all.
func TestTheRedundantStorageUnderAnItemIsRead(t *testing.T) {
health, statusX := "ONLINE\n", "pool 'storage' is healthy\n"
fs := "storage/media zfs\n"
run := func(_ context.Context, name string, args ...string) (string, error) {
switch name {
case "findmnt":
return fs, nil
case "zpool":
if args[0] == "list" {
return health, nil
}
return statusX, nil
case "cat":
return "Personalities : [raid1]\nmd127 : active raid1 sdb1[1] sda1[0]\n 976630464 blocks super 1.2 [2/1] [U_]\n\nunused devices: <none>\n", nil
}
return "", nil
}
b := &Backups{Run: run, Now: time.Now, Say: quiet}
r := b.redundancyOf(context.Background(), "/storage/media/movies")
if r == nil || r.Kind != "zfs" || r.Where != "storage" || r.Healthy == nil || !*r.Healthy {
t.Fatalf("a healthy pool: %+v", r)
}
health, statusX = "DEGRADED\n", " pool: storage\n state: DEGRADED\nstatus: One or more devices has been removed\n"
if r := b.redundancyOf(context.Background(), "/storage/media/movies"); r.Healthy == nil || *r.Healthy || !strings.Contains(r.Said, "DEGRADED") {
t.Fatalf("a degraded pool: %+v", r)
}
fs = "/dev/md127 ext4\n"
if r := b.redundancyOf(context.Background(), "/srv/x"); r == nil || r.Kind != "md" || r.Healthy == nil || *r.Healthy {
t.Fatalf("an array missing a member: %+v", r)
}
fs = "/dev/nvme0n1p3 ext4\n"
if r := b.redundancyOf(context.Background(), "/var/lib/x"); r != nil {
t.Fatalf("plain storage read as redundant: %+v", r)
}
}
// Large items are never walked: a dataset is measured from the filesystem's counters and its top-level
// entries, a shallow item from its top-level entries only; a walk that meets more than its bound stops
// and says so; and a walk runs at most once a day — the hourly round keeps the last one.
func TestNothingLargeIsWalkedAndAWalkIsBoundedAndDaily(t *testing.T) {
data := "# media\nitem movies irreplaceable /storage/media/movies - redundancy dataset\n" +
"item meta rebuildable /mnt/plex/config /mnt/plex/config backup shallow\n" +
"item big valuable /srv/big /srv/big backup\n"
where := withData(t, composed, data)
var mu sync.Mutex
var walks []string
truncated := true
run := func(_ context.Context, name string, args ...string) (string, error) {
mu.Lock()
defer mu.Unlock()
line := name + " " + strings.Join(args, " ")
switch {
case name == "test":
return "", nil
case name == "zfs" && args[len(args)-1] == "/storage/media/movies":
return "storage/media\t97067690722560\n", nil
case name == "zfs":
return "", errors.New("not a ZFS dataset")
case name == "bash" && strings.Contains(line, "-maxdepth 1"):
return "1759744800 12673\n", nil
case name == "bash":
walks = append(walks, line)
if strings.Contains(line, "-xdev") && !strings.Contains(line, "'/srv/big'") {
t.Errorf("walked something declared not to be walked: %s", line)
}
if truncated {
return fmt.Sprintf("123 1759700000 %d\n141\n", walkFiles), nil
}
return "999 1759700000 10\n0\n", nil
}
return "", nil
}
now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
b := &Backups{Where: where, Run: run, Now: func() time.Time { return now }, Say: quiet}
must(t, b.MeasureAll(context.Background(), false))
got := b.Measurements()["media"]
if m := got["movies"]; m.SizeBytes == nil || *m.SizeBytes != 97067690722560 || !strings.HasPrefix(m.Precision, "dataset storage/media") || m.LastWrite == nil {
t.Fatalf("the library from its dataset: %+v", m)
}
if m := got["meta"]; m.SizeBytes != nil || !strings.HasPrefix(m.Precision, "no size") || m.LastWrite == nil {
t.Fatalf("a shallow item: %+v", m)
}
if m := got["big"]; !strings.HasPrefix(m.Precision, "partial") {
t.Fatalf("a walk stopped at its bound: %+v", m)
}
// An hour later the hourly round reads counters again and walks nothing.
now = now.Add(time.Hour)
truncated = false
must(t, b.MeasureAll(context.Background(), false))
if len(walks) != 1 {
t.Fatalf("walked %d times in two rounds an hour apart: %v", len(walks), walks)
}
// After the night, it walks.
must(t, b.MeasureAll(context.Background(), true))
if len(walks) != 2 || b.Measurements()["media"]["big"].Precision != "exact" {
t.Fatalf("%d walks, %+v", len(walks), b.Measurements()["media"]["big"])
}
}
modules/restic/cmd/restic-backups/health_test.go
-1
View File
@@ -264,7 +264,6 @@ func TestTheManifestRunsTheNightJudgedByItsHealthTool(t *testing.T) {
env := m.Build.Artifacts[0].Env
want := map[string]struct{ run, kind, tool string }{
"night": {"night", "tool", "backup_health"},
"measure": {"measure", "unit", ""},
}
for _, r := range m.Resources {
w, ok := want[r["id"].(string)]
modules/restic/cmd/restic-backups/main.go
+28 -24
View File
@@ -24,6 +24,7 @@ import (
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/health"
)
// Seat is the role this module holds.
@@ -46,20 +47,18 @@ func main() {
}
if way == "" {
b.Emit = func(event string, body any) error { return stdio.Emit(event, body) }
b.Readings = busReadings(b.Node)
}
switch way {
case "night":
nights(b)
case "measure":
measurements(b)
case "":
if err := stdio.Serve("", tools(b)); err != nil {
say("%v", err)
os.Exit(1)
}
default:
say("%q is not a way to run; with no argument it serves its tools, with night it runs the nights, with "+
"measure it measures the data declared here", way)
say("%q is not a way to run; with no argument it serves its tools, with night it runs the nights", way)
os.Exit(2)
}
}
@@ -83,23 +82,6 @@ func nights(b *Backups) {
}
}
// measurements measures every declared item once an hour (MESH_BACKUP_MEASURE_EVERY to change it),
// the first a few minutes after start — counters hourly, a walk at most daily — and every walk after
// each night (novox/hq ADR 0233).
func measurements(b *Backups) {
every := time.Hour
if d, err := time.ParseDuration(os.Getenv("MESH_BACKUP_MEASURE_EVERY")); err == nil && d >= time.Minute {
every = d
}
time.Sleep(3 * time.Minute)
for {
if err := b.MeasureAll(context.Background(), false); err != nil {
say("measuring the data declared here failed: %v", err)
}
time.Sleep(every)
}
}
func night(b *Backups) {
ctx := context.Background()
outcome, err := b.BackUp(ctx, "")
@@ -116,9 +98,6 @@ func night(b *Backups) {
if len(failed) > 0 {
say("the night left %s without a backup", strings.Join(failed, ", "))
}
if err := b.MeasureAll(ctx, true); err != nil {
say("measuring the data declared here failed: %v", err)
}
// Sundays, the repository's own integrity with a sample of the data read back.
if time.Now().Weekday() == time.Sunday {
if err := b.Check(ctx); err != nil {
@@ -204,3 +183,28 @@ func tools(b *Backups) []stdio.Tool {
Run: func(a map[string]any) (any, error) { return b.DeletedOutcome(arg(a, "path")) }},
}
}
// busReadings reads disk-load's readings of this machine from its state, every key `<node>.<module>.<item>`.
func busReadings(node string) func() ([]health.Reading, error) {
return func() ([]health.Reading, error) {
st := stdio.State(health.ReadingsRead)
keys, err := st.Keys()
if err != nil {
return nil, err
}
var out []health.Reading
for _, k := range keys {
if !strings.HasPrefix(k, node+".") {
continue
}
e, err := st.Get(k)
if err != nil || e == nil {
continue
}
if r, err := health.DecodeReading(e.Value); err == nil {
out = append(out, r)
}
}
return out, nil
}
}
modules/restic/go.mod
+2 -2
View File
@@ -1,5 +1,5 @@
module restic
go 1.25.0
go 1.26.0
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029221656-196c5b6af277
modules/restic/go.sum
+2 -2
View File
@@ -1,2 +1,2 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c h1:l5onJwoIeH8yE/PjVlEeoPyXBsHp2NQiWLllz2fwX7s=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029221656-196c5b6af277 h1:ajjsH4MWcx45MGYASjUSRQKhLzEringuk3TA0oxZRCU=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029221656-196c5b6af277/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
modules/restic/module.json
+37 -55
View File
@@ -41,40 +41,40 @@
}
]
},
"config": {
"keep-daily": {
"type": "number",
"description": "How many nightly restore points are kept, a count from 1 to 9999: two weeks undo a mistake noticed within a fortnight. Read by the night, which thins nothing while it does not read (novox/hq issue 416); lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.",
"minimum": 1,
"maximum": 9999,
"trusted": true,
"default": 14
},
"keep-weekly": {
"type": "number",
"description": "How many weekly restore points are kept behind the dailies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.",
"minimum": 1,
"maximum": 9999,
"trusted": true,
"default": 8
},
"keep-monthly": {
"type": "number",
"description": "How many monthly restore points are kept behind the weeklies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.",
"minimum": 1,
"maximum": 9999,
"trusted": true,
"default": 6
},
"keep-hand-taken-days": {
"type": "number",
"description": "How many days a restore point taken by hand (node-backup.now, or the mesh's before a module moves) is kept at least, a count from 1 to 9999; it is kept until a later good night of its module exists, and this many days in any case, so the next thinning never takes the one taken against a risky change (novox/hq issue 453). Lowering it lets go of the older ones taken by hand at the next thinning, and that cannot be undone.",
"minimum": 1,
"maximum": 9999,
"trusted": true,
"default": 7
}
"config": {
"keep-daily": {
"type": "number",
"description": "How many nightly restore points are kept, a count from 1 to 9999: two weeks undo a mistake noticed within a fortnight. Read by the night, which thins nothing while it does not read (novox/hq issue 416); lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.",
"minimum": 1,
"maximum": 9999,
"trusted": true,
"default": 14
},
"keep-weekly": {
"type": "number",
"description": "How many weekly restore points are kept behind the dailies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.",
"minimum": 1,
"maximum": 9999,
"trusted": true,
"default": 8
},
"keep-monthly": {
"type": "number",
"description": "How many monthly restore points are kept behind the weeklies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.",
"minimum": 1,
"maximum": 9999,
"trusted": true,
"default": 6
},
"keep-hand-taken-days": {
"type": "number",
"description": "How many days a restore point taken by hand (node-backup.now, or the mesh's before a module moves) is kept at least, a count from 1 to 9999; it is kept until a later good night of its module exists, and this many days in any case, so the next thinning never takes the one taken against a risky change (novox/hq issue 453). Lowering it lets go of the older ones taken by hand at the next thinning, and that cannot be undone.",
"minimum": 1,
"maximum": 9999,
"trusted": true,
"default": 7
}
},
"resources": [
{
"id": "state",
@@ -146,27 +146,6 @@
"looks": 2,
"grace": "60s"
}
},
{
"id": "measure",
"type": "process",
"name": "restic-measure",
"artifact": "tools",
"run": [
"./restic-backups",
"measure"
],
"user": "${machine:account}",
"env": {
"MESH_BACKUP_DECLARED": "${dir:state}/backups.conf",
"MESH_BACKUP_REPOSITORY": "${dir:repository}",
"MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret",
"MESH_BACKUP_STATE": "${dir:state}",
"MESH_BACKUP_DATA": "${dir:state}/data.conf"
},
"health": {
"kind": "unit"
}
}
],
"build": {
@@ -195,5 +174,8 @@
"from": "files"
}
]
}
},
"reads": [
"disk-load.readings"
]
}