Declare every module's data; the backup holder measures it (hq ADR 0233)
Backup lines are derived from each module's data section instead of written by hand; the holder measures declared items, reads the array under them, and deletes a retired item only after a last restore point; the Go providers say each held consumer's size so an empty replacement is seen.
This commit is contained in:
jochen
@@ -151,6 +151,9 @@ func tagOf(module string) string { return "module=" + module }
|
||||
// Where is where the mesh put this module's things.
|
||||
type Where struct {
|
||||
Declared, Repository, PasswordFile, State string
|
||||
// Data is the composed file of every data item declared here (novox/hq ADR 0233); empty where the
|
||||
// module was placed by a definition older than it, and then nothing is measured.
|
||||
Data string
|
||||
}
|
||||
|
||||
func whereFromEnv() (Where, error) {
|
||||
@@ -167,6 +170,7 @@ func whereFromEnv() (Where, error) {
|
||||
Repository: get("MESH_BACKUP_REPOSITORY"),
|
||||
PasswordFile: get("MESH_BACKUP_PASSWORD_FILE"),
|
||||
State: get("MESH_BACKUP_STATE"),
|
||||
Data: os.Getenv("MESH_BACKUP_DATA"),
|
||||
}
|
||||
if len(missing) > 0 {
|
||||
return w, fmt.Errorf("%s not set; the mesh gives them to this module", strings.Join(missing, ", "))
|
||||
@@ -272,6 +276,16 @@ func (b *Backups) one(ctx context.Context, d Declared) Night {
|
||||
if err != nil {
|
||||
return fail(err)
|
||||
}
|
||||
n.Snapshot = snapshotOf(out)
|
||||
n.OK = true
|
||||
b.recordGood(d.Module, n.At)
|
||||
b.Say("%s: backed up (%s)", d.Module, n.Snapshot)
|
||||
return n
|
||||
}
|
||||
|
||||
// snapshotOf is the short id of the snapshot restic's JSON output says it made; empty when none.
|
||||
func snapshotOf(out string) string {
|
||||
id := ""
|
||||
scanner := bufio.NewScanner(strings.NewReader(out))
|
||||
scanner.Buffer(make([]byte, 1024*1024), 16*1024*1024)
|
||||
for scanner.Scan() {
|
||||
@@ -280,12 +294,10 @@ func (b *Backups) one(ctx context.Context, d Declared) Night {
|
||||
SnapshotID string `json:"snapshot_id"`
|
||||
}
|
||||
if json.Unmarshal(scanner.Bytes(), &m) == nil && m.MessageType == "summary" && len(m.SnapshotID) >= 8 {
|
||||
n.Snapshot = m.SnapshotID[:8]
|
||||
id = m.SnapshotID[:8]
|
||||
}
|
||||
}
|
||||
n.OK = true
|
||||
b.Say("%s: backed up (%s)", d.Module, n.Snapshot)
|
||||
return n
|
||||
return id
|
||||
}
|
||||
|
||||
// BackUp is a night: every module, or one, then the rotation. It answers each module's outcome.
|
||||
@@ -360,6 +372,9 @@ type ModuleBackups struct {
|
||||
LastNight *Night `json:"lastNight"`
|
||||
RestorePoints int `json:"restorePoints"`
|
||||
Newest *Snapshot `json:"newest,omitempty"`
|
||||
// Data is every item the module declares, measured, with its newest good backup (novox/hq ADR
|
||||
// 0233): what the controller's self-check reads.
|
||||
Data []ItemReport `json:"data"`
|
||||
}
|
||||
|
||||
// BackedUp is what is backed up here: each module, what it declared, its last night and its restore
|
||||
@@ -369,14 +384,23 @@ func (b *Backups) BackedUp(ctx context.Context, module string) ([]ModuleBackups,
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items, err := b.Items()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nights := b.Nights()
|
||||
good := b.Good()
|
||||
measured := b.Measurements()
|
||||
snaps, _ := b.Snapshots(ctx, module)
|
||||
out := []ModuleBackups{}
|
||||
listed := map[string]bool{}
|
||||
for _, d := range declared {
|
||||
if module != "" && d.Module != module {
|
||||
continue
|
||||
}
|
||||
m := ModuleBackups{Module: d.Module, Runs: len(d.Runs), Paths: d.Paths}
|
||||
listed[d.Module] = true
|
||||
m := ModuleBackups{Module: d.Module, Runs: len(d.Runs), Paths: d.Paths,
|
||||
Data: itemReports(items[d.Module], measured[d.Module], d.Paths, good[d.Module])}
|
||||
if n, ok := nights[d.Module]; ok {
|
||||
m.LastNight = &n
|
||||
}
|
||||
@@ -389,6 +413,13 @@ func (b *Backups) BackedUp(ctx context.Context, module string) ([]ModuleBackups,
|
||||
}
|
||||
out = append(out, m)
|
||||
}
|
||||
// A module declaring data and nothing to back up is still measured, and said.
|
||||
for name, its := range items {
|
||||
if listed[name] || (module != "" && name != module) {
|
||||
continue
|
||||
}
|
||||
out = append(out, ModuleBackups{Module: name, Paths: []string{}, Data: itemReports(its, measured[name], nil, time.Time{})})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,512 @@
|
||||
// The data the modules on this machine declare, measured (novox/hq ADR 0233).
|
||||
//
|
||||
// The mesh composes a second file beside the backup lines: every data item every module on the
|
||||
// machine declares, one line each,
|
||||
//
|
||||
// item <id> <class> <path> <covered-by> <protection>
|
||||
//
|
||||
// where covered-by is the path whose snapshot keeps it (the item itself, or the directory its dump
|
||||
// writes into), or `-` when it is not backed up, and protection is backup, redundancy, both, or none.
|
||||
// This holder measures each item that is not a cache — its size, its newest write, and the redundant
|
||||
// storage it is on and whether that is healthy (ZFS, md, btrfs) — once an hour, and says it with each
|
||||
// module's last good backup in `backed-up`. The controller keeps the readings and says when an item shrinks, stops being written,
|
||||
// goes without a backup, or is replaced by an empty copy of itself; this holder only measures.
|
||||
//
|
||||
// And it deletes, when a person has decided: an item the mesh retired — its module gone from this
|
||||
// machine — is removed by `backup_delete_retired`, and only after a last restore point of it has been
|
||||
// taken, so the deletion can be undone until a person forgets that restore point.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Item is one data item a module declares.
|
||||
type Item struct {
|
||||
Module string `json:"-"`
|
||||
ID string `json:"item"`
|
||||
Class string `json:"class"`
|
||||
Path string `json:"path"`
|
||||
CoveredBy string `json:"covered_by,omitempty"`
|
||||
Protection string `json:"protection,omitempty"`
|
||||
}
|
||||
|
||||
// Redundancy is the redundant storage an item is on, as read here.
|
||||
type Redundancy struct {
|
||||
// Kind is zfs, md or btrfs.
|
||||
Kind string `json:"kind"`
|
||||
// Where is the pool, the array device or the filesystem.
|
||||
Where string `json:"where"`
|
||||
// Healthy is nil when its state could not be read.
|
||||
Healthy *bool `json:"healthy"`
|
||||
Said string `json:"said"`
|
||||
}
|
||||
|
||||
// Measured is what one measurement found.
|
||||
type Measured struct {
|
||||
SizeBytes *int64 `json:"size_bytes,omitempty"`
|
||||
LastWrite *time.Time `json:"last_write,omitempty"`
|
||||
MeasuredAt *time.Time `json:"measured_at,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Redundancy *Redundancy `json:"redundancy,omitempty"`
|
||||
}
|
||||
|
||||
// ItemReport is one item as `backed-up` says it.
|
||||
type ItemReport struct {
|
||||
Item
|
||||
Measured
|
||||
LastBackup *time.Time `json:"last_backup,omitempty"`
|
||||
}
|
||||
|
||||
// The classes the mesh declares; a cache is listed and never measured.
|
||||
const classCache = "cache"
|
||||
|
||||
var safePath = regexp.MustCompile(`^/[^\s'"\\$` + "`" + `]*$`)
|
||||
|
||||
// parseItems reads the composed data file into each module's items. A line this holder does not read
|
||||
// is refused, naming the module, as a backup line is.
|
||||
func parseItems(text string) (map[string][]Item, error) {
|
||||
out := map[string][]Item{}
|
||||
module := ""
|
||||
for _, raw := range strings.Split(text, "\n") {
|
||||
line := strings.TrimSpace(raw)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
if m := moduleHeader.FindStringSubmatch(line); m != nil {
|
||||
module = m[1]
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(line, "#") || module == "" {
|
||||
continue
|
||||
}
|
||||
f := strings.Fields(line)
|
||||
if (len(f) != 5 && len(f) != 6) || f[0] != "item" || !safePath.MatchString(f[3]) || (f[4] != "-" && !safePath.MatchString(f[4])) {
|
||||
return nil, fmt.Errorf("%s declares a data line this holder does not read: %s", module, line)
|
||||
}
|
||||
it := Item{Module: module, ID: f[1], Class: f[2], Path: f[3]}
|
||||
if len(f) == 6 {
|
||||
it.Protection = f[5]
|
||||
}
|
||||
if f[4] != "-" {
|
||||
it.CoveredBy = f[4]
|
||||
}
|
||||
out[module] = append(out[module], it)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Items is what the modules on this machine declare; none when the mesh composed no data file — an
|
||||
// older controller, which composes none.
|
||||
func (b *Backups) Items() (map[string][]Item, error) {
|
||||
if b.Where.Data == "" {
|
||||
return map[string][]Item{}, nil
|
||||
}
|
||||
raw, err := os.ReadFile(b.Where.Data)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return map[string][]Item{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return parseItems(string(raw))
|
||||
}
|
||||
|
||||
func (b *Backups) measuredFile() string { return filepath.Join(b.Where.State, "measured.json") }
|
||||
|
||||
// Measurements is the newest measurement of each item, by module and item.
|
||||
func (b *Backups) Measurements() map[string]map[string]Measured {
|
||||
out := map[string]map[string]Measured{}
|
||||
if raw, err := os.ReadFile(b.measuredFile()); err == nil {
|
||||
_ = json.Unmarshal(raw, &out)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
var measuring sync.Mutex
|
||||
|
||||
// measureCommand sums the files under a path and finds its newest change, in one walk, as root: a
|
||||
// store's directory is often its own user's alone. One line out: "<bytes> <epoch seconds>".
|
||||
func measureCommand(path string) string {
|
||||
return "find '" + path + "' -xdev -printf '%y %s %T@\\n' 2>/dev/null | " +
|
||||
"awk 'BEGIN{s=0;m=0} {if ($1==\"f\") s+=$2; if ($3>m) m=$3} END {printf \"%d %.0f\\n\", s, m}'"
|
||||
}
|
||||
|
||||
// measure is one item, measured now.
|
||||
func (b *Backups) measure(ctx context.Context, it Item) Measured {
|
||||
at := b.Now().UTC()
|
||||
m := Measured{MeasuredAt: &at}
|
||||
if !b.exists(ctx, it.Path) {
|
||||
m.Error = it.Path + " does not exist"
|
||||
zero := int64(0)
|
||||
m.SizeBytes = &zero
|
||||
return m
|
||||
}
|
||||
out, err := b.Run(ctx, "sh", "-c", measureCommand(it.Path))
|
||||
if err != nil {
|
||||
m.Error = err.Error()
|
||||
return m
|
||||
}
|
||||
f := strings.Fields(out)
|
||||
if len(f) != 2 {
|
||||
m.Error = "the measurement said " + strings.TrimSpace(out)
|
||||
return m
|
||||
}
|
||||
size, err1 := strconv.ParseInt(f[0], 10, 64)
|
||||
epoch, err2 := strconv.ParseInt(f[1], 10, 64)
|
||||
if err1 != nil || err2 != nil {
|
||||
m.Error = "the measurement said " + strings.TrimSpace(out)
|
||||
return m
|
||||
}
|
||||
m.SizeBytes = &size
|
||||
if epoch > 0 {
|
||||
w := time.Unix(epoch, 0).UTC()
|
||||
m.LastWrite = &w
|
||||
}
|
||||
m.Redundancy = b.redundancyOf(ctx, it.Path)
|
||||
return m
|
||||
}
|
||||
|
||||
// redundancyOf is the redundant storage a path is on, read as root: a ZFS pool's health and its own
|
||||
// verdict, an md array's members, a btrfs filesystem's error counters. Nil for storage with no
|
||||
// redundancy this holder knows how to read — which, for an item said to be protected by redundancy, the
|
||||
// controller says is no protection at all.
|
||||
func (b *Backups) redundancyOf(ctx context.Context, path string) *Redundancy {
|
||||
out, err := b.Run(ctx, "findmnt", "-no", "SOURCE,FSTYPE", "--target", path)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
f := strings.Fields(out)
|
||||
if len(f) < 2 {
|
||||
return nil
|
||||
}
|
||||
source, fstype := f[0], f[1]
|
||||
yes, no := true, false
|
||||
switch {
|
||||
case fstype == "zfs":
|
||||
pool, _, _ := strings.Cut(source, "/")
|
||||
r := &Redundancy{Kind: "zfs", Where: pool}
|
||||
health, err := b.Run(ctx, "zpool", "list", "-H", "-o", "health", pool)
|
||||
if err != nil {
|
||||
r.Said = "zpool list: " + err.Error()
|
||||
return r
|
||||
}
|
||||
status, err := b.Run(ctx, "zpool", "status", "-x", pool)
|
||||
if err != nil {
|
||||
r.Said = "zpool status: " + err.Error()
|
||||
return r
|
||||
}
|
||||
health, status = strings.TrimSpace(health), strings.TrimSpace(status)
|
||||
r.Said = "health " + health + "; " + firstLineOf(status)
|
||||
if health == "ONLINE" && strings.Contains(status, "is healthy") {
|
||||
r.Healthy = &yes
|
||||
} else {
|
||||
r.Healthy = &no
|
||||
r.Said = "health " + health + "; " + oneLineOf(status)
|
||||
}
|
||||
return r
|
||||
case strings.HasPrefix(source, "/dev/md"):
|
||||
device := strings.TrimPrefix(source, "/dev/")
|
||||
r := &Redundancy{Kind: "md", Where: device}
|
||||
mdstat, err := b.Run(ctx, "cat", "/proc/mdstat")
|
||||
if err != nil {
|
||||
r.Said = err.Error()
|
||||
return r
|
||||
}
|
||||
healthy, said, found := mdHealth(mdstat, device)
|
||||
if !found {
|
||||
r.Said = device + " is not in /proc/mdstat"
|
||||
return r
|
||||
}
|
||||
r.Said = said
|
||||
if healthy {
|
||||
r.Healthy = &yes
|
||||
} else {
|
||||
r.Healthy = &no
|
||||
}
|
||||
return r
|
||||
case fstype == "btrfs":
|
||||
r := &Redundancy{Kind: "btrfs", Where: source}
|
||||
stats, err := b.Run(ctx, "btrfs", "device", "stats", "--check", path)
|
||||
if err != nil {
|
||||
r.Healthy, r.Said = &no, "device errors: "+oneLineOf(err.Error())
|
||||
return r
|
||||
}
|
||||
r.Healthy, r.Said = &yes, firstLineOf(stats)
|
||||
return r
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var mdMembers = regexp.MustCompile(`\[([U_]+)\]`)
|
||||
|
||||
// mdHealth reads one array's block of /proc/mdstat: healthy when every member is up and it is not
|
||||
// recovering.
|
||||
func mdHealth(mdstat, device string) (bool, string, bool) {
|
||||
lines := strings.Split(mdstat, "\n")
|
||||
for i, l := range lines {
|
||||
if !strings.HasPrefix(l, device+" ") {
|
||||
continue
|
||||
}
|
||||
block := l
|
||||
for j := i + 1; j < len(lines) && strings.HasPrefix(lines[j], " "); j++ {
|
||||
block += " " + strings.TrimSpace(lines[j])
|
||||
}
|
||||
members := mdMembers.FindStringSubmatch(block)
|
||||
healthy := members != nil && !strings.Contains(members[1], "_") && !strings.Contains(block, "recovery")
|
||||
return healthy, oneLineOf(block), true
|
||||
}
|
||||
return false, "", false
|
||||
}
|
||||
|
||||
func firstLineOf(s string) string {
|
||||
line, _, _ := strings.Cut(strings.TrimSpace(s), "\n")
|
||||
return line
|
||||
}
|
||||
|
||||
func oneLineOf(s string) string { return strings.Join(strings.Fields(s), " ") }
|
||||
|
||||
// MeasureAll measures every item that is not a cache, one at a time, and keeps what it found.
|
||||
func (b *Backups) MeasureAll(ctx context.Context) error {
|
||||
measuring.Lock()
|
||||
defer measuring.Unlock()
|
||||
items, err := b.Items()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
found := map[string]map[string]Measured{}
|
||||
for module, its := range items {
|
||||
for _, it := range its {
|
||||
if it.Class == classCache {
|
||||
continue
|
||||
}
|
||||
if found[module] == nil {
|
||||
found[module] = map[string]Measured{}
|
||||
}
|
||||
found[module][it.ID] = b.measure(ctx, it)
|
||||
}
|
||||
}
|
||||
raw, _ := json.MarshalIndent(found, "", " ")
|
||||
return os.WriteFile(b.measuredFile(), append(raw, '\n'), 0o600)
|
||||
}
|
||||
|
||||
func (b *Backups) goodFile() string { return filepath.Join(b.Where.State, "good.json") }
|
||||
|
||||
// Good is each module's newest good night: what a night that failed since does not erase.
|
||||
func (b *Backups) Good() map[string]time.Time {
|
||||
out := map[string]time.Time{}
|
||||
if raw, err := os.ReadFile(b.goodFile()); err == nil {
|
||||
_ = json.Unmarshal(raw, &out)
|
||||
}
|
||||
// A night recorded good before this file existed counts.
|
||||
for module, n := range b.Nights() {
|
||||
if n.OK && n.At.After(out[module]) {
|
||||
out[module] = n.At
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (b *Backups) recordGood(module string, at time.Time) {
|
||||
good := b.Good()
|
||||
good[module] = at
|
||||
raw, _ := json.MarshalIndent(good, "", " ")
|
||||
if err := os.WriteFile(b.goodFile(), append(raw, '\n'), 0o600); err != nil {
|
||||
b.Say("recording %s's good night failed: %v", module, err)
|
||||
}
|
||||
}
|
||||
|
||||
// itemReports is every item of one module, with its newest measurement and its newest good backup: the
|
||||
// module's newest good night, where that night keeps the path that covers the item.
|
||||
func itemReports(its []Item, measured map[string]Measured, paths []string, good time.Time) []ItemReport {
|
||||
out := []ItemReport{}
|
||||
for _, it := range its {
|
||||
r := ItemReport{Item: it, Measured: measured[it.ID]}
|
||||
if it.CoveredBy != "" && !good.IsZero() {
|
||||
for _, p := range paths {
|
||||
if p == it.CoveredBy {
|
||||
g := good
|
||||
r.LastBackup = &g
|
||||
}
|
||||
}
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ---- deleting a retired item -------------------------------------------------------------------
|
||||
|
||||
// Deletion is how one deletion went, by path.
|
||||
type Deletion struct {
|
||||
Module string `json:"module"`
|
||||
Item string `json:"item"`
|
||||
Started time.Time `json:"started"`
|
||||
Running bool `json:"running"`
|
||||
Done bool `json:"done"`
|
||||
OK bool `json:"ok"`
|
||||
Snapshot string `json:"snapshot,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
By string `json:"by,omitempty"`
|
||||
Why string `json:"why,omitempty"`
|
||||
}
|
||||
|
||||
func (b *Backups) deletionsFile() string { return filepath.Join(b.Where.State, "deleted.json") }
|
||||
|
||||
var deletionsMu sync.Mutex
|
||||
|
||||
func (b *Backups) deletions() map[string]Deletion {
|
||||
out := map[string]Deletion{}
|
||||
if raw, err := os.ReadFile(b.deletionsFile()); err == nil {
|
||||
_ = json.Unmarshal(raw, &out)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (b *Backups) keepDeletion(path string, d Deletion) {
|
||||
deletionsMu.Lock()
|
||||
defer deletionsMu.Unlock()
|
||||
all := b.deletions()
|
||||
all[path] = d
|
||||
raw, _ := json.MarshalIndent(all, "", " ")
|
||||
if err := os.WriteFile(b.deletionsFile(), append(raw, '\n'), 0o600); err != nil {
|
||||
b.Say("recording the deletion of %s failed: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
// refuseDeleting says why a path may not be deleted: not absolute, too near the root, or declared now
|
||||
// — by any module's item or backup line on this machine, itself, inside one, or holding one.
|
||||
func (b *Backups) refuseDeleting(path string) error {
|
||||
clean := filepath.Clean(path)
|
||||
if !safePath.MatchString(path) || clean != strings.TrimRight(path, "/") {
|
||||
return fmt.Errorf("%q is not a path this holder deletes", path)
|
||||
}
|
||||
if strings.Count(clean, "/") < 2 {
|
||||
return fmt.Errorf("%s is too near the root to be a module's data", clean)
|
||||
}
|
||||
near := func(declared string) bool {
|
||||
d := filepath.Clean(declared)
|
||||
return d == clean || strings.HasPrefix(d, clean+"/") || strings.HasPrefix(clean, d+"/")
|
||||
}
|
||||
items, err := b.Items()
|
||||
if err != nil {
|
||||
return fmt.Errorf("what is declared here cannot be read, so nothing is deleted: %v", err)
|
||||
}
|
||||
for module, its := range items {
|
||||
for _, it := range its {
|
||||
if near(it.Path) {
|
||||
return fmt.Errorf("%s is declared now — %s's %s at %s — so it is not retired; nothing is deleted",
|
||||
clean, module, it.ID, it.Path)
|
||||
}
|
||||
}
|
||||
}
|
||||
declared, err := b.Declared()
|
||||
if err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return fmt.Errorf("what is backed up here cannot be read, so nothing is deleted: %v", err)
|
||||
}
|
||||
for _, d := range declared {
|
||||
for _, p := range d.Paths {
|
||||
if near(p) {
|
||||
return fmt.Errorf("%s is backed up now as %s's %s, so it is not retired; nothing is deleted", clean, d.Module, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
if clean == filepath.Clean(b.Where.Repository) || strings.HasPrefix(b.Where.Repository, clean+"/") ||
|
||||
clean == filepath.Clean(b.Where.State) {
|
||||
return fmt.Errorf("%s holds this machine's backups; nothing is deleted", clean)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteRetired starts deleting one retired item: a last restore point of it, tagged as retired, then
|
||||
// its removal — never the removal without the restore point. Answers at once; DeletedOutcome follows
|
||||
// it. A path whose deletion already finished answers how it went.
|
||||
func (b *Backups) DeleteRetired(ctx context.Context, module, item, path, confirm, by, why string) (Deletion, error) {
|
||||
if module == "" || item == "" || path == "" {
|
||||
return Deletion{}, errors.New("module, item and path are required")
|
||||
}
|
||||
if confirm != item {
|
||||
return Deletion{}, fmt.Errorf("confirm is the item's name again (%s), to say this is meant", item)
|
||||
}
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return Deletion{}, errors.New("why is required: a deletion is a person's decision, and says why")
|
||||
}
|
||||
if d, ok := b.deletions()[path]; ok && (d.Running || (d.Done && d.OK)) {
|
||||
return d, nil
|
||||
}
|
||||
if err := b.refuseDeleting(path); err != nil {
|
||||
return Deletion{}, err
|
||||
}
|
||||
if !b.exists(ctx, path) {
|
||||
d := Deletion{Module: module, Item: item, Started: b.Now().UTC(), Done: true, OK: true,
|
||||
Error: path + " was already gone", By: by, Why: why}
|
||||
b.keepDeletion(path, d)
|
||||
return d, nil
|
||||
}
|
||||
d := Deletion{Module: module, Item: item, Started: b.Now().UTC(), Running: true, By: by, Why: why}
|
||||
b.keepDeletion(path, d)
|
||||
go b.deleteNow(context.WithoutCancel(ctx), path, d)
|
||||
return d, nil
|
||||
}
|
||||
|
||||
func (b *Backups) deleteNow(ctx context.Context, path string, d Deletion) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
finish := func(err error) {
|
||||
d.Running, d.Done = false, true
|
||||
if err != nil {
|
||||
d.Error = err.Error()
|
||||
b.Say("%s's retired %s at %s was NOT deleted: %v", d.Module, d.Item, path, err)
|
||||
} else {
|
||||
d.OK = true
|
||||
b.Say("%s's retired %s at %s DELETED by %s: %s; its last restore point is %s", d.Module, d.Item, path,
|
||||
orSomebody(d.By), d.Why, d.Snapshot)
|
||||
}
|
||||
b.keepDeletion(path, d)
|
||||
}
|
||||
if err := b.ensureRepository(ctx); err != nil {
|
||||
finish(fmt.Errorf("no restore point could be taken first: %w", err))
|
||||
return
|
||||
}
|
||||
out, err := b.restic(ctx, "backup", "--json", "--tag", tagOf(d.Module), "--tag", "retired="+d.Item, path)
|
||||
if err != nil {
|
||||
finish(fmt.Errorf("the last restore point could not be taken, so nothing was deleted: %w", err))
|
||||
return
|
||||
}
|
||||
d.Snapshot = snapshotOf(out)
|
||||
if d.Snapshot == "" {
|
||||
finish(errors.New("restic took the last restore point and named none, so nothing was deleted"))
|
||||
return
|
||||
}
|
||||
if _, err := b.Run(ctx, "rm", "-rf", "--one-file-system", "--", path); err != nil {
|
||||
finish(err)
|
||||
return
|
||||
}
|
||||
finish(nil)
|
||||
}
|
||||
|
||||
// DeletedOutcome is how the deletion of a path went.
|
||||
func (b *Backups) DeletedOutcome(path string) (Deletion, error) {
|
||||
d, ok := b.deletions()[path]
|
||||
if !ok {
|
||||
return Deletion{}, fmt.Errorf("no deletion of %s was asked of this holder", path)
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
func orSomebody(by string) string {
|
||||
if by == "" {
|
||||
return "somebody"
|
||||
}
|
||||
return by
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
package main
|
||||
|
||||
// The data the modules declare, measured, and a retired item deleted only after a last restore point
|
||||
// (novox/hq ADR 0233) — over a fake restic and a fake machine.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
const composedData = "# The data the modules on this machine declare, composed by the mesh. Do not edit.\n" +
|
||||
"# postgres\nitem store irreplaceable /var/lib/mesh-store /var/lib/mesh-store/dumps\nitem dumps rebuildable /var/lib/mesh-store/dumps -\n" +
|
||||
"# searxng\nitem valkey cache /var/lib/searxng/valkey-data -\n"
|
||||
|
||||
func withData(t *testing.T, declared, data string) Where {
|
||||
t.Helper()
|
||||
w := placed(t, declared)
|
||||
w.Data = filepath.Join(w.State, "data.conf")
|
||||
must(t, os.WriteFile(w.Data, []byte(data), 0o600))
|
||||
return w
|
||||
}
|
||||
|
||||
func TestTheComposedDataFileIsReadIntoEachModulesItems(t *testing.T) {
|
||||
got, err := parseItems(composedData)
|
||||
must(t, err)
|
||||
if len(got["postgres"]) != 2 || got["postgres"][0].CoveredBy != "/var/lib/mesh-store/dumps" || got["postgres"][1].CoveredBy != "" ||
|
||||
got["searxng"][0].Class != "cache" {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
for _, bad := range []string{"# pg\nitem a irreplaceable relative -\n", "# pg\nitem a irreplaceable /x\n", "# pg\nitem a b /x; rm -rf / -\n"} {
|
||||
if _, err := parseItems(bad); err == nil || !strings.Contains(err.Error(), "pg declares a data line") {
|
||||
t.Errorf("%q: %v", bad, err)
|
||||
}
|
||||
}
|
||||
// An older controller composes no data file: nothing is measured, nothing fails.
|
||||
b := &Backups{Where: placed(t, composed), Now: time.Now, Say: quiet}
|
||||
if items, err := b.Items(); err != nil || len(items) != 0 {
|
||||
t.Fatalf("%v, %v", items, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Every item but a cache is measured — its files' size and its newest change, in one walk as root — and
|
||||
// `backed-up` says each with the newest good night of the module that keeps the path covering it.
|
||||
func TestEveryItemButACacheIsMeasuredAndSaidWithItsLastGoodBackup(t *testing.T) {
|
||||
where := withData(t, composed, composedData)
|
||||
var mu sync.Mutex
|
||||
var walked []string
|
||||
run := func(_ context.Context, name string, args ...string) (string, error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
switch {
|
||||
case name == "test":
|
||||
return "", nil
|
||||
case name == "sh" && strings.Contains(args[1], "find '"):
|
||||
walked = append(walked, args[1])
|
||||
if strings.Contains(args[1], "'/var/lib/mesh-store'") {
|
||||
return "1073741824 1759744800\n", nil
|
||||
}
|
||||
return "5000 1759700000\n", nil
|
||||
case name == "restic":
|
||||
return "[]", nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
night := time.Date(2026, 10, 6, 3, 0, 0, 0, time.UTC)
|
||||
b := &Backups{Where: where, Run: run, Now: func() time.Time { return night.Add(time.Hour) }, Say: quiet}
|
||||
must(t, b.MeasureAll(context.Background()))
|
||||
if len(walked) != 2 {
|
||||
t.Fatalf("walked %d paths, want the two that are not a cache: %v", len(walked), walked)
|
||||
}
|
||||
b.recordGood("postgres", night)
|
||||
got, err := b.BackedUp(context.Background(), "")
|
||||
must(t, err)
|
||||
var store, dumps *ItemReport
|
||||
for _, m := range got {
|
||||
for i := range m.Data {
|
||||
switch m.Data[i].ID {
|
||||
case "store":
|
||||
store = &m.Data[i]
|
||||
case "dumps":
|
||||
dumps = &m.Data[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
if store == nil || store.SizeBytes == nil || *store.SizeBytes != 1<<30 || store.LastWrite == nil ||
|
||||
store.LastWrite.Unix() != 1759744800 || store.LastBackup == nil || !store.LastBackup.Equal(night) {
|
||||
t.Fatalf("the store is said as %+v", store)
|
||||
}
|
||||
if dumps == nil || dumps.LastBackup != nil {
|
||||
t.Fatalf("an item not backed up is said backed up: %+v", dumps)
|
||||
}
|
||||
}
|
||||
|
||||
// A retired item is deleted only after a last restore point of it, tagged as retired; a restore point
|
||||
// that fails deletes nothing; and nothing declared now — itself, inside it, or holding it — is deleted.
|
||||
func TestARetiredItemIsDeletedOnlyAfterALastRestorePoint(t *testing.T) {
|
||||
where := withData(t, composed, composedData)
|
||||
var mu sync.Mutex
|
||||
var calls []string
|
||||
failBackup := false
|
||||
run := func(_ context.Context, name string, args ...string) (string, error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
line := name + " " + strings.Join(args, " ")
|
||||
if name == "restic" {
|
||||
line = "restic " + strings.Join(args[5:], " ")
|
||||
}
|
||||
calls = append(calls, line)
|
||||
switch {
|
||||
case name == "test":
|
||||
return "", nil
|
||||
case strings.HasPrefix(line, "restic backup"):
|
||||
if failBackup {
|
||||
return "", errors.New("the repository is locked")
|
||||
}
|
||||
return "{\"message_type\":\"summary\",\"snapshot_id\":\"0123456789abcdef\"}\n", nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
b := &Backups{Where: where, Run: run, Now: time.Now, Say: quiet}
|
||||
ctx := context.Background()
|
||||
for _, refused := range []struct{ path, want string }{
|
||||
{"/var/lib/mesh-store", "declared now"},
|
||||
{"/var/lib/mesh-store/dumps/old", "declared now"},
|
||||
{"/var/lib", "declared now"},
|
||||
{"/var/lib/mailu/data-mail", "backed up now"},
|
||||
{"/srv", "too near the root"},
|
||||
{"relative/x", "not a path"},
|
||||
} {
|
||||
if _, err := b.DeleteRetired(ctx, "m", "i", refused.path, "i", "op", "tidy"); err == nil || !strings.Contains(err.Error(), refused.want) {
|
||||
t.Errorf("%s: %v, want %q", refused.path, err, refused.want)
|
||||
}
|
||||
}
|
||||
if _, err := b.DeleteRetired(ctx, "house", "config", "/var/lib/house/config", "nope", "op", "tidy"); err == nil {
|
||||
t.Error("a deletion without the item's name again was started")
|
||||
}
|
||||
if _, err := b.DeleteRetired(ctx, "house", "config", "/var/lib/house/config", "config", "op", ""); err == nil {
|
||||
t.Error("a deletion without why was started")
|
||||
}
|
||||
|
||||
wait := func(path string) Deletion {
|
||||
t.Helper()
|
||||
for i := 0; i < 200; i++ {
|
||||
if d, err := b.DeletedOutcome(path); err == nil && d.Done {
|
||||
return d
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("the deletion of %s never finished", path)
|
||||
return Deletion{}
|
||||
}
|
||||
|
||||
mu.Lock()
|
||||
failBackup = true
|
||||
mu.Unlock()
|
||||
_, err := b.DeleteRetired(ctx, "house", "config", "/var/lib/house/config", "config", "op", "moved to the anchor")
|
||||
must(t, err)
|
||||
if d := wait("/var/lib/house/config"); d.OK || !strings.Contains(d.Error, "nothing was deleted") {
|
||||
t.Fatalf("a deletion with no restore point: %+v", d)
|
||||
}
|
||||
mu.Lock()
|
||||
for _, c := range calls {
|
||||
if strings.HasPrefix(c, "rm ") {
|
||||
t.Fatal("it deleted without a last restore point")
|
||||
}
|
||||
}
|
||||
mu.Unlock()
|
||||
|
||||
mu.Lock()
|
||||
failBackup, calls = false, nil
|
||||
mu.Unlock()
|
||||
_, err = b.DeleteRetired(ctx, "house", "config", "/var/lib/house/config", "config", "op", "moved to the anchor")
|
||||
must(t, err)
|
||||
d := wait("/var/lib/house/config")
|
||||
if !d.OK || d.Snapshot != "01234567" {
|
||||
t.Fatalf("%+v", d)
|
||||
}
|
||||
var backup, rm = -1, -1
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
for i, c := range calls {
|
||||
switch {
|
||||
case c == "restic backup --json --tag module=house --tag retired=config /var/lib/house/config":
|
||||
backup = i
|
||||
case c == "rm -rf --one-file-system -- /var/lib/house/config":
|
||||
rm = i
|
||||
}
|
||||
}
|
||||
if backup < 0 || rm < 0 || rm < backup {
|
||||
t.Fatalf("ran %v", calls)
|
||||
}
|
||||
// Asked again, it answers how it went rather than starting over.
|
||||
again, err := b.DeleteRetired(ctx, "house", "config", "/var/lib/house/config", "config", "op", "moved")
|
||||
if err != nil || !again.Done || !again.OK {
|
||||
t.Fatalf("%+v, %v", again, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The redundant storage under an item is read: a ZFS pool healthy, then degraded; an md array with a
|
||||
// member missing; and plain storage, which is no redundancy at all.
|
||||
func TestTheRedundantStorageUnderAnItemIsRead(t *testing.T) {
|
||||
health, statusX := "ONLINE\n", "pool 'storage' is healthy\n"
|
||||
fs := "storage/media zfs\n"
|
||||
run := func(_ context.Context, name string, args ...string) (string, error) {
|
||||
switch name {
|
||||
case "findmnt":
|
||||
return fs, nil
|
||||
case "zpool":
|
||||
if args[0] == "list" {
|
||||
return health, nil
|
||||
}
|
||||
return statusX, nil
|
||||
case "cat":
|
||||
return "Personalities : [raid1]\nmd127 : active raid1 sdb1[1] sda1[0]\n 976630464 blocks super 1.2 [2/1] [U_]\n\nunused devices: <none>\n", nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
b := &Backups{Run: run, Now: time.Now, Say: quiet}
|
||||
r := b.redundancyOf(context.Background(), "/storage/media/movies")
|
||||
if r == nil || r.Kind != "zfs" || r.Where != "storage" || r.Healthy == nil || !*r.Healthy {
|
||||
t.Fatalf("a healthy pool: %+v", r)
|
||||
}
|
||||
health, statusX = "DEGRADED\n", " pool: storage\n state: DEGRADED\nstatus: One or more devices has been removed\n"
|
||||
if r := b.redundancyOf(context.Background(), "/storage/media/movies"); r.Healthy == nil || *r.Healthy || !strings.Contains(r.Said, "DEGRADED") {
|
||||
t.Fatalf("a degraded pool: %+v", r)
|
||||
}
|
||||
fs = "/dev/md127 ext4\n"
|
||||
if r := b.redundancyOf(context.Background(), "/srv/x"); r == nil || r.Kind != "md" || r.Healthy == nil || *r.Healthy {
|
||||
t.Fatalf("an array missing a member: %+v", r)
|
||||
}
|
||||
fs = "/dev/nvme0n1p3 ext4\n"
|
||||
if r := b.redundancyOf(context.Background(), "/var/lib/x"); r != nil {
|
||||
t.Fatalf("plain storage read as redundant: %+v", r)
|
||||
}
|
||||
}
|
||||
@@ -32,6 +32,7 @@ func main() {
|
||||
}
|
||||
b := &Backups{Where: where, Run: execRunner, Now: time.Now, Say: say}
|
||||
go nights(b)
|
||||
go measurements(b)
|
||||
if err := stdio.Serve("", tools(b)); err != nil {
|
||||
say("%v", err)
|
||||
os.Exit(1)
|
||||
@@ -56,6 +57,22 @@ func nights(b *Backups) {
|
||||
}
|
||||
}
|
||||
|
||||
// measurements measures every declared item once an hour (MESH_BACKUP_MEASURE_EVERY to change it),
|
||||
// the first a few minutes after start, and after every night (novox/hq ADR 0233).
|
||||
func measurements(b *Backups) {
|
||||
every := time.Hour
|
||||
if d, err := time.ParseDuration(os.Getenv("MESH_BACKUP_MEASURE_EVERY")); err == nil && d >= time.Minute {
|
||||
every = d
|
||||
}
|
||||
time.Sleep(3 * time.Minute)
|
||||
for {
|
||||
if err := b.MeasureAll(context.Background()); err != nil {
|
||||
say("measuring the data declared here failed: %v", err)
|
||||
}
|
||||
time.Sleep(every)
|
||||
}
|
||||
}
|
||||
|
||||
func night(b *Backups) {
|
||||
ctx := context.Background()
|
||||
outcome, err := b.BackUp(ctx, "")
|
||||
@@ -72,6 +89,9 @@ func night(b *Backups) {
|
||||
if len(failed) > 0 {
|
||||
say("the night left %s without a backup", strings.Join(failed, ", "))
|
||||
}
|
||||
if err := b.MeasureAll(ctx); err != nil {
|
||||
say("measuring the data declared here failed: %v", err)
|
||||
}
|
||||
// Sundays, the repository's own integrity with a sample of the data read back.
|
||||
if time.Now().Weekday() == time.Sunday {
|
||||
if err := b.Check(ctx); err != nil {
|
||||
@@ -133,5 +153,22 @@ func tools(b *Backups) []stdio.Tool {
|
||||
}
|
||||
return b.Restore(context.Background(), module, arg(a, "snapshot"), arg(a, "path"))
|
||||
}),
|
||||
// Deleting a retired item, when a person decided (novox/hq ADR 0233): the controller's `cleanup
|
||||
// delete` asks these. Not seat verbs — nobody else calls them.
|
||||
{Name: "backup_delete_retired", Description: "Delete one item of data the mesh retired on this machine — " +
|
||||
"its module is gone from here — after taking a last restore point of it, tagged retired, so it can be " +
|
||||
"restored until that restore point is forgotten. Refuses anything declared now. Answers at once; " +
|
||||
"backup_deleted says how it went. Use the controller's `cleanup delete`, which records the hand act.",
|
||||
Input: map[string]any{"module": str("the module it was"), "item": str("the item"), "path": str("where it is"),
|
||||
"confirm": str("the item's name again"), "why": str("why — required"), "by": str("who decided"),
|
||||
"via": str("mesh-controller when asked through its verbs")},
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
return b.DeleteRetired(context.Background(), arg(a, "module"), arg(a, "item"), arg(a, "path"),
|
||||
arg(a, "confirm"), arg(a, "by"), arg(a, "why"))
|
||||
}},
|
||||
{Name: "backup_deleted", Description: "How the deletion of one retired item went: running, or done with " +
|
||||
"its last restore point, or refused with why.",
|
||||
Input: map[string]any{"path": str("where it is")},
|
||||
Run: func(a map[string]any) (any, error) { return b.DeletedOutcome(arg(a, "path")) }},
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user