diff --git a/modules/restic/cmd/restic-backups/backups.go b/modules/restic/cmd/restic-backups/backups.go index 03a4e598..8f428db4 100644 --- a/modules/restic/cmd/restic-backups/backups.go +++ b/modules/restic/cmd/restic-backups/backups.go @@ -25,6 +25,7 @@ import ( "encoding/json" "errors" "fmt" + "git.novox.be/novox/mesh-sdk/go/health" "os" "os/exec" "path/filepath" @@ -188,6 +189,9 @@ type Backups struct { // measuring run as processes of their own). Node is the machine's name, said in the events. Emit func(event string, body any) error Node string + // Readings is every reading disk-load keeps for this machine (its state `readings`, ADR 0571 clause 3); nil where the + // process has no runtime to read it. + Readings func() ([]health.Reading, error) } // The locks, files in this module's state. The night runs in a process of its own and `now`, `restore` and a diff --git a/modules/restic/cmd/restic-backups/data.go b/modules/restic/cmd/restic-backups/data.go index a7a3d5fc..8c597684 100644 --- a/modules/restic/cmd/restic-backups/data.go +++ b/modules/restic/cmd/restic-backups/data.go @@ -1,26 +1,9 @@ -// The data the modules on this machine declare, measured (novox/hq ADR 0233). +// The data the modules on this machine declare (novox/hq ADR 0233), and what the readings say of them. // -// The mesh composes a second file beside the backup lines: every data item every module on the -// machine declares, one line each, -// -// item -// -// where covered-by is the path whose snapshot keeps it (the item itself, or the directory its dump -// writes into), or `-` when it is not backed up; protection is backup, redundancy, both, or none; and -// measure is how the item is measured. This holder measures each item that is not a cache — its size, -// its newest write, and the redundant storage it is on and whether that is healthy (ZFS, md, btrfs) — -// and says it, with each module's last good backup and the precision of the measurement, in -// `backed-up`. -// -// **Never an unbounded walk of something large.** Three methods, the item's own choice: -// -// - `walk` (the default, for small items): every file summed and the newest change found, in one walk -// as root — at most once a day, and stopped after walkFor or walkFiles, said as "measured partially"; -// - `dataset`: the ZFS dataset holding the path — its `used` from the filesystem's own counters — and -// the newest change among the path's top-level entries; hourly, and nothing is walked; -// - `shallow`: the newest change among the top-level entries only, and no size; hourly. The controller keeps the readings and says when an item shrinks, stops being written, -// -// goes without a backup, or is replaced by an empty copy of itself; this holder only measures. +// The mesh composes a file beside the backup lines: every data item every module on the machine declares, one line +// each, which mesh-sdk reads (catalogue.ParseDataItems). disk-load measures the items and keeps each reading and its +// seven-day peak as its own state (ADR 0571 clause 3); this holder reads them there and says each, with each module's +// last good backup, in `backed-up`. // // And it deletes, when a person has decided: an item the mesh retired — its module gone from this // machine — is removed by `backup_delete_retired`, and only after a last restore point of it has been @@ -32,104 +15,32 @@ import ( "encoding/json" "errors" "fmt" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/health" "os" "path/filepath" "regexp" - "strconv" "strings" "sync" "time" ) -// Item is one data item a module declares. -type Item struct { - // Measure is walk, dataset or shallow. - Measure string `json:"measure,omitempty"` - Module string `json:"-"` - ID string `json:"item"` - Class string `json:"class"` - Path string `json:"path"` - CoveredBy string `json:"covered_by,omitempty"` - Protection string `json:"protection,omitempty"` -} +// Item is one data item a module declares, as placed on this machine (mesh-sdk's reading of the holder's data text). +type Item = catalogue.PlacedDataItem -// Redundancy is the redundant storage an item is on, as read here. -type Redundancy struct { - // Kind is zfs, md or btrfs. - Kind string `json:"kind"` - // Where is the pool, the array device or the filesystem. - Where string `json:"where"` - // Healthy is nil when its state could not be read. - Healthy *bool `json:"healthy"` - Said string `json:"said"` -} - -// Measured is what one measurement found. -type Measured struct { - SizeBytes *int64 `json:"size_bytes,omitempty"` - LastWrite *time.Time `json:"last_write,omitempty"` - MeasuredAt *time.Time `json:"measured_at,omitempty"` - // Precision says what the size is: "exact", "dataset : its whole size", "partial: …" (a lower - // bound, never compared), or "no size: …". - Precision string `json:"precision,omitempty"` - Error string `json:"error,omitempty"` - Redundancy *Redundancy `json:"redundancy,omitempty"` -} - -// ItemReport is one item as `backed-up` says it. +// ItemReport is one item as `backed-up` says it: the item, its newest reading from disk-load (ADR 0571 clause 3) and +// its newest good backup. type ItemReport struct { Item - Measured + health.Measured + // Peak is the largest size in the last seven days, on PeakDay. + Peak *int64 `json:"peak_bytes,omitempty"` + PeakDay string `json:"peak_day,omitempty"` LastBackup *time.Time `json:"last_backup,omitempty"` } -// The classes the mesh declares; a cache is listed and never measured. -const classCache = "cache" - var safePath = regexp.MustCompile(`^/[^\s'"\\$` + "`" + `]*$`) -// parseItems reads the composed data file into each module's items. A line this holder does not read -// is refused, naming the module, as a backup line is. -func parseItems(text string) (map[string][]Item, error) { - out := map[string][]Item{} - module := "" - for _, raw := range strings.Split(text, "\n") { - line := strings.TrimSpace(raw) - if line == "" { - continue - } - if m := moduleHeader.FindStringSubmatch(line); m != nil { - module = m[1] - continue - } - if strings.HasPrefix(line, "#") || module == "" { - continue - } - f := strings.Fields(line) - if len(f) < 5 || len(f) > 7 || f[0] != "item" || !safePath.MatchString(f[3]) || (f[4] != "-" && !safePath.MatchString(f[4])) { - return nil, fmt.Errorf("%s declares a data line this holder does not read: %s", module, line) - } - it := Item{Module: module, ID: f[1], Class: f[2], Path: f[3]} - if len(f) >= 6 { - it.Protection = f[5] - } - it.Measure = measureWalk - if len(f) == 7 { - it.Measure = f[6] - } - switch it.Measure { - case measureWalk, measureDataset, measureShallow: - default: - return nil, fmt.Errorf("%s declares a data line this holder does not read: %s", module, line) - } - if f[4] != "-" { - it.CoveredBy = f[4] - } - out[module] = append(out[module], it) - } - return out, nil -} - // Items is what the modules on this machine declare; none when the mesh composed no data file — an // older controller, which composes none. func (b *Backups) Items() (map[string][]Item, error) { @@ -143,257 +54,30 @@ func (b *Backups) Items() (map[string][]Item, error) { if err != nil { return nil, err } - return parseItems(string(raw)) + return catalogue.ParseDataItems(string(raw)) } -func (b *Backups) measuredFile() string { return filepath.Join(b.Where.State, "measured.json") } - -// Measurements is the newest measurement of each item, by module and item. -func (b *Backups) Measurements() map[string]map[string]Measured { - out := map[string]map[string]Measured{} - if raw, err := os.ReadFile(b.measuredFile()); err == nil { - _ = json.Unmarshal(raw, &out) +// Measurements is the newest reading disk-load keeps of each item on this machine, by module and item (its state +// `readings`, ADR 0571 clause 3); none where it cannot be read, which `backed-up` then says as no reading. +func (b *Backups) Measurements() map[string]map[string]health.Reading { + out := map[string]map[string]health.Reading{} + if b.Readings == nil { + return out + } + rs, err := b.Readings() + if err != nil { + b.Say("disk-load's readings could not be read: %v", err) + return out + } + for _, r := range rs { + if out[r.Module] == nil { + out[r.Module] = map[string]health.Reading{} + } + out[r.Module][r.ID] = r } return out } -// The ways an item is measured. -const ( - measureWalk = "walk" - measureDataset = "dataset" - measureShallow = "shallow" -) - -// The bounds on a walk, and how often one runs: a walk is for small items, and one that meets a large -// one stops and says so rather than loading the disks for hours. -var ( - walkFor = 10 * time.Minute - walkFiles = 2_000_000 - walkEvery = 23 * time.Hour -) - -// walkCommand sums the files under a path and finds its newest change, in one walk, as root — bounded -// by time and by files. One line out: " ". -func walkCommand(path string) string { - return fmt.Sprintf("timeout %d find '%s' -xdev -printf '%%y %%s %%T@\\n' 2>/dev/null | head -n %d | "+ - "awk 'BEGIN{s=0;m=0;n=0} {n++; if ($1==\"f\") s+=$2; if ($3>m) m=$3} END {printf \"%%d %%.0f %%d\\n\", s, m, n}'; "+ - "echo \"${PIPESTATUS[0]:-0}\"", int(walkFor.Seconds()), path, walkFiles) -} - -// topCommand is the newest change among a path and its top-level entries, and how many there are: -// one directory read, bounded. " ". -func topCommand(path string) string { - return "timeout 60 find '" + path + "' -maxdepth 1 -printf '%T@\\n' 2>/dev/null | " + - "awk 'BEGIN{m=0;n=0} {n++; if ($1>m) m=$1} END {printf \"%.0f %d\\n\", m, n-1}'" -} - -func epochTime(s string) *time.Time { - epoch, err := strconv.ParseInt(s, 10, 64) - if err != nil || epoch <= 0 { - return nil - } - t := time.Unix(epoch, 0).UTC() - return &t -} - -// measure is one item, measured now, by its own method. -func (b *Backups) measure(ctx context.Context, it Item) Measured { - at := b.Now().UTC() - m := Measured{MeasuredAt: &at} - if !b.exists(ctx, it.Path) { - m.Error = it.Path + " does not exist" - zero := int64(0) - m.SizeBytes, m.Precision = &zero, "exact" - return m - } - m.Redundancy = b.redundancyOf(ctx, it.Path) - switch it.Measure { - case measureDataset, measureShallow: - out, err := b.Run(ctx, "bash", "-c", topCommand(it.Path)) - if f := strings.Fields(out); err == nil && len(f) == 2 { - m.LastWrite = epochTime(f[0]) - } else if err != nil { - m.Error = err.Error() - } - m.Precision = "no size: the newest change among its top-level entries only" - if it.Measure == measureShallow { - return m - } - out, err = b.Run(ctx, "zfs", "list", "-Hp", "-o", "name,used", it.Path) - f := strings.Fields(out) - if err != nil || len(f) != 2 { - m.Precision = "no size: it is on no ZFS dataset to read one from; the newest change among its top-level entries only" - return m - } - used, err := strconv.ParseInt(f[1], 10, 64) - if err != nil { - return m - } - m.SizeBytes = &used - m.Precision = "dataset " + f[0] + ": its whole size, from the filesystem's counters; the newest change among the top-level entries" - return m - } - out, err := b.Run(ctx, "bash", "-c", walkCommand(it.Path)) - if err != nil { - m.Error = err.Error() - return m - } - lines := strings.Split(strings.TrimSpace(out), "\n") - f := strings.Fields(lines[0]) - if len(f) != 3 || len(lines) < 2 { - m.Error = "the measurement said " + strings.TrimSpace(out) - return m - } - size, err1 := strconv.ParseInt(f[0], 10, 64) - files, err2 := strconv.Atoi(f[2]) - if err1 != nil || err2 != nil { - m.Error = "the measurement said " + strings.TrimSpace(out) - return m - } - m.SizeBytes, m.LastWrite = &size, epochTime(f[1]) - m.Precision = "exact" - if status := strings.TrimSpace(lines[len(lines)-1]); status == "124" || files >= walkFiles { - m.Precision = fmt.Sprintf("partial: measured partially — stopped after %s or %d files; the size is a lower bound, "+ - "and this item wants measure: dataset or shallow", walkFor, walkFiles) - } - return m -} - -// redundancyOf is the redundant storage a path is on, read as root: a ZFS pool's health and its own -// verdict, an md array's members, a btrfs filesystem's error counters. Nil for storage with no -// redundancy this holder knows how to read — which, for an item said to be protected by redundancy, the -// controller says is no protection at all. -func (b *Backups) redundancyOf(ctx context.Context, path string) *Redundancy { - out, err := b.Run(ctx, "findmnt", "-no", "SOURCE,FSTYPE", "--target", path) - if err != nil { - return nil - } - f := strings.Fields(out) - if len(f) < 2 { - return nil - } - source, fstype := f[0], f[1] - yes, no := true, false - switch { - case fstype == "zfs": - pool, _, _ := strings.Cut(source, "/") - r := &Redundancy{Kind: "zfs", Where: pool} - health, err := b.Run(ctx, "zpool", "list", "-H", "-o", "health", pool) - if err != nil { - r.Said = "zpool list: " + err.Error() - return r - } - status, err := b.Run(ctx, "zpool", "status", "-x", pool) - if err != nil { - r.Said = "zpool status: " + err.Error() - return r - } - health, status = strings.TrimSpace(health), strings.TrimSpace(status) - r.Said = "health " + health + "; " + firstLineOf(status) - if health == "ONLINE" && strings.Contains(status, "is healthy") { - r.Healthy = &yes - } else { - r.Healthy = &no - r.Said = "health " + health + "; " + oneLineOf(status) - } - return r - case strings.HasPrefix(source, "/dev/md"): - device := strings.TrimPrefix(source, "/dev/") - r := &Redundancy{Kind: "md", Where: device} - mdstat, err := b.Run(ctx, "cat", "/proc/mdstat") - if err != nil { - r.Said = err.Error() - return r - } - healthy, said, found := mdHealth(mdstat, device) - if !found { - r.Said = device + " is not in /proc/mdstat" - return r - } - r.Said = said - if healthy { - r.Healthy = &yes - } else { - r.Healthy = &no - } - return r - case fstype == "btrfs": - r := &Redundancy{Kind: "btrfs", Where: source} - stats, err := b.Run(ctx, "btrfs", "device", "stats", "--check", path) - if err != nil { - r.Healthy, r.Said = &no, "device errors: "+oneLineOf(err.Error()) - return r - } - r.Healthy, r.Said = &yes, firstLineOf(stats) - return r - } - return nil -} - -var mdMembers = regexp.MustCompile(`\[([U_]+)\]`) - -// mdHealth reads one array's block of /proc/mdstat: healthy when every member is up and it is not -// recovering. -func mdHealth(mdstat, device string) (bool, string, bool) { - lines := strings.Split(mdstat, "\n") - for i, l := range lines { - if !strings.HasPrefix(l, device+" ") { - continue - } - block := l - for j := i + 1; j < len(lines) && strings.HasPrefix(lines[j], " "); j++ { - block += " " + strings.TrimSpace(lines[j]) - } - members := mdMembers.FindStringSubmatch(block) - healthy := members != nil && !strings.Contains(members[1], "_") && !strings.Contains(block, "recovery") - return healthy, oneLineOf(block), true - } - return false, "", false -} - -func firstLineOf(s string) string { - line, _, _ := strings.Cut(strings.TrimSpace(s), "\n") - return line -} - -func oneLineOf(s string) string { return strings.Join(strings.Fields(s), " ") } - -// MeasureAll measures every item that is not a cache, one at a time, and keeps what it found. An item -// measured by a walk is walked only when walks is true or its last walk is older than walkEvery: the -// hourly round reads counters, and a walk runs at most once a day. -func (b *Backups) MeasureAll(ctx context.Context, walks bool) error { - // The hourly round runs in its own process and the walk after a night in the night's (novox/hq issue 454). - release, err := b.hold(measureLock) - if err != nil { - return err - } - defer release() - items, err := b.Items() - if err != nil { - return err - } - before := b.Measurements() - found := map[string]map[string]Measured{} - for module, its := range items { - for _, it := range its { - if it.Class == classCache { - continue - } - if found[module] == nil { - found[module] = map[string]Measured{} - } - if was, ok := before[module][it.ID]; ok && it.Measure == measureWalk && !walks && was.MeasuredAt != nil && - b.Now().Sub(*was.MeasuredAt) < walkEvery { - found[module][it.ID] = was - continue - } - found[module][it.ID] = b.measure(ctx, it) - } - } - raw, _ := json.MarshalIndent(found, "", " ") - return os.WriteFile(b.measuredFile(), append(raw, '\n'), 0o600) -} - func (b *Backups) goodFile() string { return filepath.Join(b.Where.State, "good.json") } // Good is each module's newest good night: what a night that failed since does not erase. @@ -420,12 +104,15 @@ func (b *Backups) recordGood(module string, at time.Time) { } } -// itemReports is every item of one module, with its newest measurement and its newest good backup: the +// itemReports is every item of one module, with its newest reading and its newest good backup: the // module's newest good night, where that night keeps the path that covers the item. -func itemReports(its []Item, measured map[string]Measured, paths []string, good time.Time) []ItemReport { +func itemReports(its []Item, measured map[string]health.Reading, paths []string, good time.Time) []ItemReport { out := []ItemReport{} for _, it := range its { - r := ItemReport{Item: it, Measured: measured[it.ID]} + r := ItemReport{Item: it} + if m, ok := measured[it.ID]; ok { + r.Measured, r.Peak, r.PeakDay = m.Measured, m.Peak, m.PeakAt + } if it.CoveredBy != "" && !good.IsZero() { for _, p := range paths { if p == it.CoveredBy { diff --git a/modules/restic/cmd/restic-backups/data_test.go b/modules/restic/cmd/restic-backups/data_test.go index 0834ad61..aee683b3 100644 --- a/modules/restic/cmd/restic-backups/data_test.go +++ b/modules/restic/cmd/restic-backups/data_test.go @@ -6,7 +6,8 @@ package main import ( "context" "errors" - "fmt" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/health" "os" "path/filepath" "strings" @@ -28,53 +29,34 @@ func withData(t *testing.T, declared, data string) Where { } func TestTheComposedDataFileIsReadIntoEachModulesItems(t *testing.T) { - got, err := parseItems(composedData) + b := &Backups{Where: withData(t, composed, composedData), Now: time.Now, Say: quiet} + got, err := b.Items() must(t, err) if len(got["postgres"]) != 2 || got["postgres"][0].CoveredBy != "/var/lib/mesh-store/dumps" || got["postgres"][1].CoveredBy != "" || got["searxng"][0].Class != "cache" { t.Fatalf("%+v", got) } - for _, bad := range []string{"# pg\nitem a irreplaceable relative -\n", "# pg\nitem a irreplaceable /x\n", "# pg\nitem a b /x; rm -rf / -\n"} { - if _, err := parseItems(bad); err == nil || !strings.Contains(err.Error(), "pg declares a data line") { - t.Errorf("%q: %v", bad, err) - } - } - // An older controller composes no data file: nothing is measured, nothing fails. - b := &Backups{Where: placed(t, composed), Now: time.Now, Say: quiet} + // An older controller composes no data file: nothing is declared, nothing fails. + b = &Backups{Where: placed(t, composed), Now: time.Now, Say: quiet} if items, err := b.Items(); err != nil || len(items) != 0 { t.Fatalf("%v, %v", items, err) } } -// Every item but a cache is measured — its files' size and its newest change, in one walk as root — and -// `backed-up` says each with the newest good night of the module that keeps the path covering it. -func TestEveryItemButACacheIsMeasuredAndSaidWithItsLastGoodBackup(t *testing.T) { +// `backed-up` says every item with disk-load's newest reading and its peak, and the newest good night of the module that +// keeps the path covering it; an item with no reading yet is said with none. +func TestEveryItemIsSaidWithItsReadingAndItsLastGoodBackup(t *testing.T) { where := withData(t, composed, composedData) - var mu sync.Mutex - var walked []string - run := func(_ context.Context, name string, args ...string) (string, error) { - mu.Lock() - defer mu.Unlock() - switch { - case name == "test": - return "", nil - case name == "bash" && strings.Contains(args[1], "find '"): - walked = append(walked, args[1]) - if strings.Contains(args[1], "'/var/lib/mesh-store'") { - return "1073741824 1759744800 4000\n0\n", nil - } - return "5000 1759700000 3\n0\n", nil - case name == "restic": - return "[]", nil - } - return "", nil - } + size, peak := int64(1<<30), int64(2<<30) + at := time.Date(2026, 10, 6, 4, 0, 0, 0, time.UTC) + write := time.Unix(1759744800, 0).UTC() night := time.Date(2026, 10, 6, 3, 0, 0, 0, time.UTC) - b := &Backups{Where: where, Run: run, Now: func() time.Time { return night.Add(time.Hour) }, Say: quiet} - must(t, b.MeasureAll(context.Background(), true)) - if len(walked) != 2 { - t.Fatalf("walked %d paths, want the two that are not a cache: %v", len(walked), walked) - } + b := &Backups{Where: where, Now: func() time.Time { return night.Add(time.Hour) }, Say: quiet, + Run: func(context.Context, string, ...string) (string, error) { return "[]", nil }, + Readings: func() ([]health.Reading, error) { + return []health.Reading{{V: 1, Node: "ace", PlacedDataItem: catalogue.PlacedDataItem{Module: "postgres", ID: "store"}, + Measured: health.Measured{SizeBytes: &size, LastWrite: &write, MeasuredAt: &at, Precision: "exact"}, Peak: &peak, PeakAt: "2026-10-05"}}, nil + }} b.recordGood("postgres", night) got, err := b.BackedUp(context.Background(), "") must(t, err) @@ -89,12 +71,12 @@ func TestEveryItemButACacheIsMeasuredAndSaidWithItsLastGoodBackup(t *testing.T) } } } - if store == nil || store.SizeBytes == nil || *store.SizeBytes != 1<<30 || store.LastWrite == nil || - store.LastWrite.Unix() != 1759744800 || store.LastBackup == nil || !store.LastBackup.Equal(night) { + if store == nil || store.SizeBytes == nil || *store.SizeBytes != 1<<30 || store.Peak == nil || *store.Peak != 2<<30 || + store.LastWrite == nil || store.LastWrite.Unix() != 1759744800 || store.LastBackup == nil || !store.LastBackup.Equal(night) { t.Fatalf("the store is said as %+v", store) } - if dumps == nil || dumps.LastBackup != nil { - t.Fatalf("an item not backed up is said backed up: %+v", dumps) + if dumps == nil || dumps.LastBackup != nil || dumps.SizeBytes != nil { + t.Fatalf("an item with no reading or no backup is said with one: %+v", dumps) } } @@ -202,104 +184,3 @@ func TestARetiredItemIsDeletedOnlyAfterALastRestorePoint(t *testing.T) { t.Fatalf("%+v, %v", again, err) } } - -// The redundant storage under an item is read: a ZFS pool healthy, then degraded; an md array with a -// member missing; and plain storage, which is no redundancy at all. -func TestTheRedundantStorageUnderAnItemIsRead(t *testing.T) { - health, statusX := "ONLINE\n", "pool 'storage' is healthy\n" - fs := "storage/media zfs\n" - run := func(_ context.Context, name string, args ...string) (string, error) { - switch name { - case "findmnt": - return fs, nil - case "zpool": - if args[0] == "list" { - return health, nil - } - return statusX, nil - case "cat": - return "Personalities : [raid1]\nmd127 : active raid1 sdb1[1] sda1[0]\n 976630464 blocks super 1.2 [2/1] [U_]\n\nunused devices: \n", nil - } - return "", nil - } - b := &Backups{Run: run, Now: time.Now, Say: quiet} - r := b.redundancyOf(context.Background(), "/storage/media/movies") - if r == nil || r.Kind != "zfs" || r.Where != "storage" || r.Healthy == nil || !*r.Healthy { - t.Fatalf("a healthy pool: %+v", r) - } - health, statusX = "DEGRADED\n", " pool: storage\n state: DEGRADED\nstatus: One or more devices has been removed\n" - if r := b.redundancyOf(context.Background(), "/storage/media/movies"); r.Healthy == nil || *r.Healthy || !strings.Contains(r.Said, "DEGRADED") { - t.Fatalf("a degraded pool: %+v", r) - } - fs = "/dev/md127 ext4\n" - if r := b.redundancyOf(context.Background(), "/srv/x"); r == nil || r.Kind != "md" || r.Healthy == nil || *r.Healthy { - t.Fatalf("an array missing a member: %+v", r) - } - fs = "/dev/nvme0n1p3 ext4\n" - if r := b.redundancyOf(context.Background(), "/var/lib/x"); r != nil { - t.Fatalf("plain storage read as redundant: %+v", r) - } -} - -// Large items are never walked: a dataset is measured from the filesystem's counters and its top-level -// entries, a shallow item from its top-level entries only; a walk that meets more than its bound stops -// and says so; and a walk runs at most once a day — the hourly round keeps the last one. -func TestNothingLargeIsWalkedAndAWalkIsBoundedAndDaily(t *testing.T) { - data := "# media\nitem movies irreplaceable /storage/media/movies - redundancy dataset\n" + - "item meta rebuildable /mnt/plex/config /mnt/plex/config backup shallow\n" + - "item big valuable /srv/big /srv/big backup\n" - where := withData(t, composed, data) - var mu sync.Mutex - var walks []string - truncated := true - run := func(_ context.Context, name string, args ...string) (string, error) { - mu.Lock() - defer mu.Unlock() - line := name + " " + strings.Join(args, " ") - switch { - case name == "test": - return "", nil - case name == "zfs" && args[len(args)-1] == "/storage/media/movies": - return "storage/media\t97067690722560\n", nil - case name == "zfs": - return "", errors.New("not a ZFS dataset") - case name == "bash" && strings.Contains(line, "-maxdepth 1"): - return "1759744800 12673\n", nil - case name == "bash": - walks = append(walks, line) - if strings.Contains(line, "-xdev") && !strings.Contains(line, "'/srv/big'") { - t.Errorf("walked something declared not to be walked: %s", line) - } - if truncated { - return fmt.Sprintf("123 1759700000 %d\n141\n", walkFiles), nil - } - return "999 1759700000 10\n0\n", nil - } - return "", nil - } - now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) - b := &Backups{Where: where, Run: run, Now: func() time.Time { return now }, Say: quiet} - must(t, b.MeasureAll(context.Background(), false)) - got := b.Measurements()["media"] - if m := got["movies"]; m.SizeBytes == nil || *m.SizeBytes != 97067690722560 || !strings.HasPrefix(m.Precision, "dataset storage/media") || m.LastWrite == nil { - t.Fatalf("the library from its dataset: %+v", m) - } - if m := got["meta"]; m.SizeBytes != nil || !strings.HasPrefix(m.Precision, "no size") || m.LastWrite == nil { - t.Fatalf("a shallow item: %+v", m) - } - if m := got["big"]; !strings.HasPrefix(m.Precision, "partial") { - t.Fatalf("a walk stopped at its bound: %+v", m) - } - // An hour later the hourly round reads counters again and walks nothing. - now = now.Add(time.Hour) - truncated = false - must(t, b.MeasureAll(context.Background(), false)) - if len(walks) != 1 { - t.Fatalf("walked %d times in two rounds an hour apart: %v", len(walks), walks) - } - // After the night, it walks. - must(t, b.MeasureAll(context.Background(), true)) - if len(walks) != 2 || b.Measurements()["media"]["big"].Precision != "exact" { - t.Fatalf("%d walks, %+v", len(walks), b.Measurements()["media"]["big"]) - } -} diff --git a/modules/restic/cmd/restic-backups/health_test.go b/modules/restic/cmd/restic-backups/health_test.go index 838aa88e..9518ab4f 100644 --- a/modules/restic/cmd/restic-backups/health_test.go +++ b/modules/restic/cmd/restic-backups/health_test.go @@ -264,7 +264,6 @@ func TestTheManifestRunsTheNightJudgedByItsHealthTool(t *testing.T) { env := m.Build.Artifacts[0].Env want := map[string]struct{ run, kind, tool string }{ "night": {"night", "tool", "backup_health"}, - "measure": {"measure", "unit", ""}, } for _, r := range m.Resources { w, ok := want[r["id"].(string)] diff --git a/modules/restic/cmd/restic-backups/main.go b/modules/restic/cmd/restic-backups/main.go index f4e08e32..ce5b7525 100644 --- a/modules/restic/cmd/restic-backups/main.go +++ b/modules/restic/cmd/restic-backups/main.go @@ -24,6 +24,7 @@ import ( "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/health" ) // Seat is the role this module holds. @@ -46,20 +47,18 @@ func main() { } if way == "" { b.Emit = func(event string, body any) error { return stdio.Emit(event, body) } + b.Readings = busReadings(b.Node) } switch way { case "night": nights(b) - case "measure": - measurements(b) case "": if err := stdio.Serve("", tools(b)); err != nil { say("%v", err) os.Exit(1) } default: - say("%q is not a way to run; with no argument it serves its tools, with night it runs the nights, with "+ - "measure it measures the data declared here", way) + say("%q is not a way to run; with no argument it serves its tools, with night it runs the nights", way) os.Exit(2) } } @@ -83,23 +82,6 @@ func nights(b *Backups) { } } -// measurements measures every declared item once an hour (MESH_BACKUP_MEASURE_EVERY to change it), -// the first a few minutes after start — counters hourly, a walk at most daily — and every walk after -// each night (novox/hq ADR 0233). -func measurements(b *Backups) { - every := time.Hour - if d, err := time.ParseDuration(os.Getenv("MESH_BACKUP_MEASURE_EVERY")); err == nil && d >= time.Minute { - every = d - } - time.Sleep(3 * time.Minute) - for { - if err := b.MeasureAll(context.Background(), false); err != nil { - say("measuring the data declared here failed: %v", err) - } - time.Sleep(every) - } -} - func night(b *Backups) { ctx := context.Background() outcome, err := b.BackUp(ctx, "") @@ -116,9 +98,6 @@ func night(b *Backups) { if len(failed) > 0 { say("the night left %s without a backup", strings.Join(failed, ", ")) } - if err := b.MeasureAll(ctx, true); err != nil { - say("measuring the data declared here failed: %v", err) - } // Sundays, the repository's own integrity with a sample of the data read back. if time.Now().Weekday() == time.Sunday { if err := b.Check(ctx); err != nil { @@ -204,3 +183,28 @@ func tools(b *Backups) []stdio.Tool { Run: func(a map[string]any) (any, error) { return b.DeletedOutcome(arg(a, "path")) }}, } } + +// busReadings reads disk-load's readings of this machine from its state, every key `..`. +func busReadings(node string) func() ([]health.Reading, error) { + return func() ([]health.Reading, error) { + st := stdio.State(health.ReadingsRead) + keys, err := st.Keys() + if err != nil { + return nil, err + } + var out []health.Reading + for _, k := range keys { + if !strings.HasPrefix(k, node+".") { + continue + } + e, err := st.Get(k) + if err != nil || e == nil { + continue + } + if r, err := health.DecodeReading(e.Value); err == nil { + out = append(out, r) + } + } + return out, nil + } +} diff --git a/modules/restic/go.mod b/modules/restic/go.mod index 7ec16d84..27304285 100644 --- a/modules/restic/go.mod +++ b/modules/restic/go.mod @@ -1,5 +1,5 @@ module restic -go 1.25.0 +go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029221656-196c5b6af277 diff --git a/modules/restic/go.sum b/modules/restic/go.sum index dfc73823..fc894705 100644 --- a/modules/restic/go.sum +++ b/modules/restic/go.sum @@ -1,2 +1,2 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c h1:l5onJwoIeH8yE/PjVlEeoPyXBsHp2NQiWLllz2fwX7s= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029221656-196c5b6af277 h1:ajjsH4MWcx45MGYASjUSRQKhLzEringuk3TA0oxZRCU= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029221656-196c5b6af277/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= diff --git a/modules/restic/module.json b/modules/restic/module.json index ea5ec8b9..283bfef9 100644 --- a/modules/restic/module.json +++ b/modules/restic/module.json @@ -41,40 +41,40 @@ } ] }, - "config": { - "keep-daily": { - "type": "number", - "description": "How many nightly restore points are kept, a count from 1 to 9999: two weeks undo a mistake noticed within a fortnight. Read by the night, which thins nothing while it does not read (novox/hq issue 416); lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", - "minimum": 1, - "maximum": 9999, - "trusted": true, - "default": 14 - }, - "keep-weekly": { - "type": "number", - "description": "How many weekly restore points are kept behind the dailies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", - "minimum": 1, - "maximum": 9999, - "trusted": true, - "default": 8 - }, - "keep-monthly": { - "type": "number", - "description": "How many monthly restore points are kept behind the weeklies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", - "minimum": 1, - "maximum": 9999, - "trusted": true, - "default": 6 - }, - "keep-hand-taken-days": { - "type": "number", - "description": "How many days a restore point taken by hand (node-backup.now, or the mesh's before a module moves) is kept at least, a count from 1 to 9999; it is kept until a later good night of its module exists, and this many days in any case, so the next thinning never takes the one taken against a risky change (novox/hq issue 453). Lowering it lets go of the older ones taken by hand at the next thinning, and that cannot be undone.", - "minimum": 1, - "maximum": 9999, - "trusted": true, - "default": 7 - } + "config": { + "keep-daily": { + "type": "number", + "description": "How many nightly restore points are kept, a count from 1 to 9999: two weeks undo a mistake noticed within a fortnight. Read by the night, which thins nothing while it does not read (novox/hq issue 416); lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", + "minimum": 1, + "maximum": 9999, + "trusted": true, + "default": 14 }, + "keep-weekly": { + "type": "number", + "description": "How many weekly restore points are kept behind the dailies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", + "minimum": 1, + "maximum": 9999, + "trusted": true, + "default": 8 + }, + "keep-monthly": { + "type": "number", + "description": "How many monthly restore points are kept behind the weeklies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", + "minimum": 1, + "maximum": 9999, + "trusted": true, + "default": 6 + }, + "keep-hand-taken-days": { + "type": "number", + "description": "How many days a restore point taken by hand (node-backup.now, or the mesh's before a module moves) is kept at least, a count from 1 to 9999; it is kept until a later good night of its module exists, and this many days in any case, so the next thinning never takes the one taken against a risky change (novox/hq issue 453). Lowering it lets go of the older ones taken by hand at the next thinning, and that cannot be undone.", + "minimum": 1, + "maximum": 9999, + "trusted": true, + "default": 7 + } + }, "resources": [ { "id": "state", @@ -146,27 +146,6 @@ "looks": 2, "grace": "60s" } - }, - { - "id": "measure", - "type": "process", - "name": "restic-measure", - "artifact": "tools", - "run": [ - "./restic-backups", - "measure" - ], - "user": "${machine:account}", - "env": { - "MESH_BACKUP_DECLARED": "${dir:state}/backups.conf", - "MESH_BACKUP_REPOSITORY": "${dir:repository}", - "MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret", - "MESH_BACKUP_STATE": "${dir:state}", - "MESH_BACKUP_DATA": "${dir:state}/data.conf" - }, - "health": { - "kind": "unit" - } } ], "build": { @@ -195,5 +174,8 @@ "from": "files" } ] - } + }, + "reads": [ + "disk-load.readings" + ] }