restic: read each item's reading and peak from disk-load's state and the item list from the sdk; the measuring leaves the backup holder (issue 1955, ADR 0571 clause 3)
This commit is contained in:
@@ -1,26 +1,9 @@
|
||||
// The data the modules on this machine declare, measured (novox/hq ADR 0233).
|
||||
// The data the modules on this machine declare (novox/hq ADR 0233), and what the readings say of them.
|
||||
//
|
||||
// The mesh composes a second file beside the backup lines: every data item every module on the
|
||||
// machine declares, one line each,
|
||||
//
|
||||
// item <id> <class> <path> <covered-by> <protection> <measure>
|
||||
//
|
||||
// where covered-by is the path whose snapshot keeps it (the item itself, or the directory its dump
|
||||
// writes into), or `-` when it is not backed up; protection is backup, redundancy, both, or none; and
|
||||
// measure is how the item is measured. This holder measures each item that is not a cache — its size,
|
||||
// its newest write, and the redundant storage it is on and whether that is healthy (ZFS, md, btrfs) —
|
||||
// and says it, with each module's last good backup and the precision of the measurement, in
|
||||
// `backed-up`.
|
||||
//
|
||||
// **Never an unbounded walk of something large.** Three methods, the item's own choice:
|
||||
//
|
||||
// - `walk` (the default, for small items): every file summed and the newest change found, in one walk
|
||||
// as root — at most once a day, and stopped after walkFor or walkFiles, said as "measured partially";
|
||||
// - `dataset`: the ZFS dataset holding the path — its `used` from the filesystem's own counters — and
|
||||
// the newest change among the path's top-level entries; hourly, and nothing is walked;
|
||||
// - `shallow`: the newest change among the top-level entries only, and no size; hourly. The controller keeps the readings and says when an item shrinks, stops being written,
|
||||
//
|
||||
// goes without a backup, or is replaced by an empty copy of itself; this holder only measures.
|
||||
// The mesh composes a file beside the backup lines: every data item every module on the machine declares, one line
|
||||
// each, which mesh-sdk reads (catalogue.ParseDataItems). disk-load measures the items and keeps each reading and its
|
||||
// seven-day peak as its own state (ADR 0571 clause 3); this holder reads them there and says each, with each module's
|
||||
// last good backup, in `backed-up`.
|
||||
//
|
||||
// And it deletes, when a person has decided: an item the mesh retired — its module gone from this
|
||||
// machine — is removed by `backup_delete_retired`, and only after a last restore point of it has been
|
||||
@@ -32,104 +15,32 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Item is one data item a module declares.
|
||||
type Item struct {
|
||||
// Measure is walk, dataset or shallow.
|
||||
Measure string `json:"measure,omitempty"`
|
||||
Module string `json:"-"`
|
||||
ID string `json:"item"`
|
||||
Class string `json:"class"`
|
||||
Path string `json:"path"`
|
||||
CoveredBy string `json:"covered_by,omitempty"`
|
||||
Protection string `json:"protection,omitempty"`
|
||||
}
|
||||
// Item is one data item a module declares, as placed on this machine (mesh-sdk's reading of the holder's data text).
|
||||
type Item = catalogue.PlacedDataItem
|
||||
|
||||
// Redundancy is the redundant storage an item is on, as read here.
|
||||
type Redundancy struct {
|
||||
// Kind is zfs, md or btrfs.
|
||||
Kind string `json:"kind"`
|
||||
// Where is the pool, the array device or the filesystem.
|
||||
Where string `json:"where"`
|
||||
// Healthy is nil when its state could not be read.
|
||||
Healthy *bool `json:"healthy"`
|
||||
Said string `json:"said"`
|
||||
}
|
||||
|
||||
// Measured is what one measurement found.
|
||||
type Measured struct {
|
||||
SizeBytes *int64 `json:"size_bytes,omitempty"`
|
||||
LastWrite *time.Time `json:"last_write,omitempty"`
|
||||
MeasuredAt *time.Time `json:"measured_at,omitempty"`
|
||||
// Precision says what the size is: "exact", "dataset <name>: its whole size", "partial: …" (a lower
|
||||
// bound, never compared), or "no size: …".
|
||||
Precision string `json:"precision,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Redundancy *Redundancy `json:"redundancy,omitempty"`
|
||||
}
|
||||
|
||||
// ItemReport is one item as `backed-up` says it.
|
||||
// ItemReport is one item as `backed-up` says it: the item, its newest reading from disk-load (ADR 0571 clause 3) and
|
||||
// its newest good backup.
|
||||
type ItemReport struct {
|
||||
Item
|
||||
Measured
|
||||
health.Measured
|
||||
// Peak is the largest size in the last seven days, on PeakDay.
|
||||
Peak *int64 `json:"peak_bytes,omitempty"`
|
||||
PeakDay string `json:"peak_day,omitempty"`
|
||||
LastBackup *time.Time `json:"last_backup,omitempty"`
|
||||
}
|
||||
|
||||
// The classes the mesh declares; a cache is listed and never measured.
|
||||
const classCache = "cache"
|
||||
|
||||
var safePath = regexp.MustCompile(`^/[^\s'"\\$` + "`" + `]*$`)
|
||||
|
||||
// parseItems reads the composed data file into each module's items. A line this holder does not read
|
||||
// is refused, naming the module, as a backup line is.
|
||||
func parseItems(text string) (map[string][]Item, error) {
|
||||
out := map[string][]Item{}
|
||||
module := ""
|
||||
for _, raw := range strings.Split(text, "\n") {
|
||||
line := strings.TrimSpace(raw)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
if m := moduleHeader.FindStringSubmatch(line); m != nil {
|
||||
module = m[1]
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(line, "#") || module == "" {
|
||||
continue
|
||||
}
|
||||
f := strings.Fields(line)
|
||||
if len(f) < 5 || len(f) > 7 || f[0] != "item" || !safePath.MatchString(f[3]) || (f[4] != "-" && !safePath.MatchString(f[4])) {
|
||||
return nil, fmt.Errorf("%s declares a data line this holder does not read: %s", module, line)
|
||||
}
|
||||
it := Item{Module: module, ID: f[1], Class: f[2], Path: f[3]}
|
||||
if len(f) >= 6 {
|
||||
it.Protection = f[5]
|
||||
}
|
||||
it.Measure = measureWalk
|
||||
if len(f) == 7 {
|
||||
it.Measure = f[6]
|
||||
}
|
||||
switch it.Measure {
|
||||
case measureWalk, measureDataset, measureShallow:
|
||||
default:
|
||||
return nil, fmt.Errorf("%s declares a data line this holder does not read: %s", module, line)
|
||||
}
|
||||
if f[4] != "-" {
|
||||
it.CoveredBy = f[4]
|
||||
}
|
||||
out[module] = append(out[module], it)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Items is what the modules on this machine declare; none when the mesh composed no data file — an
|
||||
// older controller, which composes none.
|
||||
func (b *Backups) Items() (map[string][]Item, error) {
|
||||
@@ -143,257 +54,30 @@ func (b *Backups) Items() (map[string][]Item, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return parseItems(string(raw))
|
||||
return catalogue.ParseDataItems(string(raw))
|
||||
}
|
||||
|
||||
func (b *Backups) measuredFile() string { return filepath.Join(b.Where.State, "measured.json") }
|
||||
|
||||
// Measurements is the newest measurement of each item, by module and item.
|
||||
func (b *Backups) Measurements() map[string]map[string]Measured {
|
||||
out := map[string]map[string]Measured{}
|
||||
if raw, err := os.ReadFile(b.measuredFile()); err == nil {
|
||||
_ = json.Unmarshal(raw, &out)
|
||||
// Measurements is the newest reading disk-load keeps of each item on this machine, by module and item (its state
|
||||
// `readings`, ADR 0571 clause 3); none where it cannot be read, which `backed-up` then says as no reading.
|
||||
func (b *Backups) Measurements() map[string]map[string]health.Reading {
|
||||
out := map[string]map[string]health.Reading{}
|
||||
if b.Readings == nil {
|
||||
return out
|
||||
}
|
||||
rs, err := b.Readings()
|
||||
if err != nil {
|
||||
b.Say("disk-load's readings could not be read: %v", err)
|
||||
return out
|
||||
}
|
||||
for _, r := range rs {
|
||||
if out[r.Module] == nil {
|
||||
out[r.Module] = map[string]health.Reading{}
|
||||
}
|
||||
out[r.Module][r.ID] = r
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// The ways an item is measured.
|
||||
const (
|
||||
measureWalk = "walk"
|
||||
measureDataset = "dataset"
|
||||
measureShallow = "shallow"
|
||||
)
|
||||
|
||||
// The bounds on a walk, and how often one runs: a walk is for small items, and one that meets a large
|
||||
// one stops and says so rather than loading the disks for hours.
|
||||
var (
|
||||
walkFor = 10 * time.Minute
|
||||
walkFiles = 2_000_000
|
||||
walkEvery = 23 * time.Hour
|
||||
)
|
||||
|
||||
// walkCommand sums the files under a path and finds its newest change, in one walk, as root — bounded
|
||||
// by time and by files. One line out: "<bytes> <epoch seconds> <files> <complete 1|0>".
|
||||
func walkCommand(path string) string {
|
||||
return fmt.Sprintf("timeout %d find '%s' -xdev -printf '%%y %%s %%T@\\n' 2>/dev/null | head -n %d | "+
|
||||
"awk 'BEGIN{s=0;m=0;n=0} {n++; if ($1==\"f\") s+=$2; if ($3>m) m=$3} END {printf \"%%d %%.0f %%d\\n\", s, m, n}'; "+
|
||||
"echo \"${PIPESTATUS[0]:-0}\"", int(walkFor.Seconds()), path, walkFiles)
|
||||
}
|
||||
|
||||
// topCommand is the newest change among a path and its top-level entries, and how many there are:
|
||||
// one directory read, bounded. "<epoch seconds> <entries>".
|
||||
func topCommand(path string) string {
|
||||
return "timeout 60 find '" + path + "' -maxdepth 1 -printf '%T@\\n' 2>/dev/null | " +
|
||||
"awk 'BEGIN{m=0;n=0} {n++; if ($1>m) m=$1} END {printf \"%.0f %d\\n\", m, n-1}'"
|
||||
}
|
||||
|
||||
func epochTime(s string) *time.Time {
|
||||
epoch, err := strconv.ParseInt(s, 10, 64)
|
||||
if err != nil || epoch <= 0 {
|
||||
return nil
|
||||
}
|
||||
t := time.Unix(epoch, 0).UTC()
|
||||
return &t
|
||||
}
|
||||
|
||||
// measure is one item, measured now, by its own method.
|
||||
func (b *Backups) measure(ctx context.Context, it Item) Measured {
|
||||
at := b.Now().UTC()
|
||||
m := Measured{MeasuredAt: &at}
|
||||
if !b.exists(ctx, it.Path) {
|
||||
m.Error = it.Path + " does not exist"
|
||||
zero := int64(0)
|
||||
m.SizeBytes, m.Precision = &zero, "exact"
|
||||
return m
|
||||
}
|
||||
m.Redundancy = b.redundancyOf(ctx, it.Path)
|
||||
switch it.Measure {
|
||||
case measureDataset, measureShallow:
|
||||
out, err := b.Run(ctx, "bash", "-c", topCommand(it.Path))
|
||||
if f := strings.Fields(out); err == nil && len(f) == 2 {
|
||||
m.LastWrite = epochTime(f[0])
|
||||
} else if err != nil {
|
||||
m.Error = err.Error()
|
||||
}
|
||||
m.Precision = "no size: the newest change among its top-level entries only"
|
||||
if it.Measure == measureShallow {
|
||||
return m
|
||||
}
|
||||
out, err = b.Run(ctx, "zfs", "list", "-Hp", "-o", "name,used", it.Path)
|
||||
f := strings.Fields(out)
|
||||
if err != nil || len(f) != 2 {
|
||||
m.Precision = "no size: it is on no ZFS dataset to read one from; the newest change among its top-level entries only"
|
||||
return m
|
||||
}
|
||||
used, err := strconv.ParseInt(f[1], 10, 64)
|
||||
if err != nil {
|
||||
return m
|
||||
}
|
||||
m.SizeBytes = &used
|
||||
m.Precision = "dataset " + f[0] + ": its whole size, from the filesystem's counters; the newest change among the top-level entries"
|
||||
return m
|
||||
}
|
||||
out, err := b.Run(ctx, "bash", "-c", walkCommand(it.Path))
|
||||
if err != nil {
|
||||
m.Error = err.Error()
|
||||
return m
|
||||
}
|
||||
lines := strings.Split(strings.TrimSpace(out), "\n")
|
||||
f := strings.Fields(lines[0])
|
||||
if len(f) != 3 || len(lines) < 2 {
|
||||
m.Error = "the measurement said " + strings.TrimSpace(out)
|
||||
return m
|
||||
}
|
||||
size, err1 := strconv.ParseInt(f[0], 10, 64)
|
||||
files, err2 := strconv.Atoi(f[2])
|
||||
if err1 != nil || err2 != nil {
|
||||
m.Error = "the measurement said " + strings.TrimSpace(out)
|
||||
return m
|
||||
}
|
||||
m.SizeBytes, m.LastWrite = &size, epochTime(f[1])
|
||||
m.Precision = "exact"
|
||||
if status := strings.TrimSpace(lines[len(lines)-1]); status == "124" || files >= walkFiles {
|
||||
m.Precision = fmt.Sprintf("partial: measured partially — stopped after %s or %d files; the size is a lower bound, "+
|
||||
"and this item wants measure: dataset or shallow", walkFor, walkFiles)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// redundancyOf is the redundant storage a path is on, read as root: a ZFS pool's health and its own
|
||||
// verdict, an md array's members, a btrfs filesystem's error counters. Nil for storage with no
|
||||
// redundancy this holder knows how to read — which, for an item said to be protected by redundancy, the
|
||||
// controller says is no protection at all.
|
||||
func (b *Backups) redundancyOf(ctx context.Context, path string) *Redundancy {
|
||||
out, err := b.Run(ctx, "findmnt", "-no", "SOURCE,FSTYPE", "--target", path)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
f := strings.Fields(out)
|
||||
if len(f) < 2 {
|
||||
return nil
|
||||
}
|
||||
source, fstype := f[0], f[1]
|
||||
yes, no := true, false
|
||||
switch {
|
||||
case fstype == "zfs":
|
||||
pool, _, _ := strings.Cut(source, "/")
|
||||
r := &Redundancy{Kind: "zfs", Where: pool}
|
||||
health, err := b.Run(ctx, "zpool", "list", "-H", "-o", "health", pool)
|
||||
if err != nil {
|
||||
r.Said = "zpool list: " + err.Error()
|
||||
return r
|
||||
}
|
||||
status, err := b.Run(ctx, "zpool", "status", "-x", pool)
|
||||
if err != nil {
|
||||
r.Said = "zpool status: " + err.Error()
|
||||
return r
|
||||
}
|
||||
health, status = strings.TrimSpace(health), strings.TrimSpace(status)
|
||||
r.Said = "health " + health + "; " + firstLineOf(status)
|
||||
if health == "ONLINE" && strings.Contains(status, "is healthy") {
|
||||
r.Healthy = &yes
|
||||
} else {
|
||||
r.Healthy = &no
|
||||
r.Said = "health " + health + "; " + oneLineOf(status)
|
||||
}
|
||||
return r
|
||||
case strings.HasPrefix(source, "/dev/md"):
|
||||
device := strings.TrimPrefix(source, "/dev/")
|
||||
r := &Redundancy{Kind: "md", Where: device}
|
||||
mdstat, err := b.Run(ctx, "cat", "/proc/mdstat")
|
||||
if err != nil {
|
||||
r.Said = err.Error()
|
||||
return r
|
||||
}
|
||||
healthy, said, found := mdHealth(mdstat, device)
|
||||
if !found {
|
||||
r.Said = device + " is not in /proc/mdstat"
|
||||
return r
|
||||
}
|
||||
r.Said = said
|
||||
if healthy {
|
||||
r.Healthy = &yes
|
||||
} else {
|
||||
r.Healthy = &no
|
||||
}
|
||||
return r
|
||||
case fstype == "btrfs":
|
||||
r := &Redundancy{Kind: "btrfs", Where: source}
|
||||
stats, err := b.Run(ctx, "btrfs", "device", "stats", "--check", path)
|
||||
if err != nil {
|
||||
r.Healthy, r.Said = &no, "device errors: "+oneLineOf(err.Error())
|
||||
return r
|
||||
}
|
||||
r.Healthy, r.Said = &yes, firstLineOf(stats)
|
||||
return r
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var mdMembers = regexp.MustCompile(`\[([U_]+)\]`)
|
||||
|
||||
// mdHealth reads one array's block of /proc/mdstat: healthy when every member is up and it is not
|
||||
// recovering.
|
||||
func mdHealth(mdstat, device string) (bool, string, bool) {
|
||||
lines := strings.Split(mdstat, "\n")
|
||||
for i, l := range lines {
|
||||
if !strings.HasPrefix(l, device+" ") {
|
||||
continue
|
||||
}
|
||||
block := l
|
||||
for j := i + 1; j < len(lines) && strings.HasPrefix(lines[j], " "); j++ {
|
||||
block += " " + strings.TrimSpace(lines[j])
|
||||
}
|
||||
members := mdMembers.FindStringSubmatch(block)
|
||||
healthy := members != nil && !strings.Contains(members[1], "_") && !strings.Contains(block, "recovery")
|
||||
return healthy, oneLineOf(block), true
|
||||
}
|
||||
return false, "", false
|
||||
}
|
||||
|
||||
func firstLineOf(s string) string {
|
||||
line, _, _ := strings.Cut(strings.TrimSpace(s), "\n")
|
||||
return line
|
||||
}
|
||||
|
||||
func oneLineOf(s string) string { return strings.Join(strings.Fields(s), " ") }
|
||||
|
||||
// MeasureAll measures every item that is not a cache, one at a time, and keeps what it found. An item
|
||||
// measured by a walk is walked only when walks is true or its last walk is older than walkEvery: the
|
||||
// hourly round reads counters, and a walk runs at most once a day.
|
||||
func (b *Backups) MeasureAll(ctx context.Context, walks bool) error {
|
||||
// The hourly round runs in its own process and the walk after a night in the night's (novox/hq issue 454).
|
||||
release, err := b.hold(measureLock)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer release()
|
||||
items, err := b.Items()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
before := b.Measurements()
|
||||
found := map[string]map[string]Measured{}
|
||||
for module, its := range items {
|
||||
for _, it := range its {
|
||||
if it.Class == classCache {
|
||||
continue
|
||||
}
|
||||
if found[module] == nil {
|
||||
found[module] = map[string]Measured{}
|
||||
}
|
||||
if was, ok := before[module][it.ID]; ok && it.Measure == measureWalk && !walks && was.MeasuredAt != nil &&
|
||||
b.Now().Sub(*was.MeasuredAt) < walkEvery {
|
||||
found[module][it.ID] = was
|
||||
continue
|
||||
}
|
||||
found[module][it.ID] = b.measure(ctx, it)
|
||||
}
|
||||
}
|
||||
raw, _ := json.MarshalIndent(found, "", " ")
|
||||
return os.WriteFile(b.measuredFile(), append(raw, '\n'), 0o600)
|
||||
}
|
||||
|
||||
func (b *Backups) goodFile() string { return filepath.Join(b.Where.State, "good.json") }
|
||||
|
||||
// Good is each module's newest good night: what a night that failed since does not erase.
|
||||
@@ -420,12 +104,15 @@ func (b *Backups) recordGood(module string, at time.Time) {
|
||||
}
|
||||
}
|
||||
|
||||
// itemReports is every item of one module, with its newest measurement and its newest good backup: the
|
||||
// itemReports is every item of one module, with its newest reading and its newest good backup: the
|
||||
// module's newest good night, where that night keeps the path that covers the item.
|
||||
func itemReports(its []Item, measured map[string]Measured, paths []string, good time.Time) []ItemReport {
|
||||
func itemReports(its []Item, measured map[string]health.Reading, paths []string, good time.Time) []ItemReport {
|
||||
out := []ItemReport{}
|
||||
for _, it := range its {
|
||||
r := ItemReport{Item: it, Measured: measured[it.ID]}
|
||||
r := ItemReport{Item: it}
|
||||
if m, ok := measured[it.ID]; ok {
|
||||
r.Measured, r.Peak, r.PeakDay = m.Measured, m.Peak, m.PeakAt
|
||||
}
|
||||
if it.CoveredBy != "" && !good.IsZero() {
|
||||
for _, p := range paths {
|
||||
if p == it.CoveredBy {
|
||||
|
||||
Reference in New Issue
Block a user