mesh-catalog/modules/restic/cmd/restic-backups/data.go

// The data the modules on this machine declare, measured (novox/hq ADR 0233).
//
// The mesh composes a second file beside the backup lines: every data item every module on the
// machine declares, one line each,
//
//	item <id> <class> <path> <covered-by> <protection>
//
// where covered-by is the path whose snapshot keeps it (the item itself, or the directory its dump
// writes into), or `-` when it is not backed up, and protection is backup, redundancy, both, or none.
// This holder measures each item that is not a cache — its size, its newest write, and the redundant
// storage it is on and whether that is healthy (ZFS, md, btrfs) — once an hour, and says it with each
// module's last good backup in `backed-up`. The controller keeps the readings and says when an item shrinks, stops being written,
// goes without a backup, or is replaced by an empty copy of itself; this holder only measures.
//
// And it deletes, when a person has decided: an item the mesh retired — its module gone from this
// machine — is removed by `backup_delete_retired`, and only after a last restore point of it has been
// taken, so the deletion can be undone until a person forgets that restore point.
package main

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"
)

// Item is one data item a module declares.
type Item struct {
	Module     string `json:"-"`
	ID         string `json:"item"`
	Class      string `json:"class"`
	Path       string `json:"path"`
	CoveredBy  string `json:"covered_by,omitempty"`
	Protection string `json:"protection,omitempty"`
}

// Redundancy is the redundant storage an item is on, as read here.
type Redundancy struct {
	// Kind is zfs, md or btrfs.
	Kind string `json:"kind"`
	// Where is the pool, the array device or the filesystem.
	Where string `json:"where"`
	// Healthy is nil when its state could not be read.
	Healthy *bool  `json:"healthy"`
	Said    string `json:"said"`
}

// Measured is what one measurement found.
type Measured struct {
	SizeBytes  *int64      `json:"size_bytes,omitempty"`
	LastWrite  *time.Time  `json:"last_write,omitempty"`
	MeasuredAt *time.Time  `json:"measured_at,omitempty"`
	Error      string      `json:"error,omitempty"`
	Redundancy *Redundancy `json:"redundancy,omitempty"`
}

// ItemReport is one item as `backed-up` says it.
type ItemReport struct {
	Item
	Measured
	LastBackup *time.Time `json:"last_backup,omitempty"`
}

// The classes the mesh declares; a cache is listed and never measured.
const classCache = "cache"

var safePath = regexp.MustCompile(`^/[^\s'"\\$` + "`" + `]*$`)

// parseItems reads the composed data file into each module's items. A line this holder does not read
// is refused, naming the module, as a backup line is.
func parseItems(text string) (map[string][]Item, error) {
	out := map[string][]Item{}
	module := ""
	for _, raw := range strings.Split(text, "\n") {
		line := strings.TrimSpace(raw)
		if line == "" {
			continue
		}
		if m := moduleHeader.FindStringSubmatch(line); m != nil {
			module = m[1]
			continue
		}
		if strings.HasPrefix(line, "#") || module == "" {
			continue
		}
		f := strings.Fields(line)
		if (len(f) != 5 && len(f) != 6) || f[0] != "item" || !safePath.MatchString(f[3]) || (f[4] != "-" && !safePath.MatchString(f[4])) {
			return nil, fmt.Errorf("%s declares a data line this holder does not read: %s", module, line)
		}
		it := Item{Module: module, ID: f[1], Class: f[2], Path: f[3]}
		if len(f) == 6 {
			it.Protection = f[5]
		}
		if f[4] != "-" {
			it.CoveredBy = f[4]
		}
		out[module] = append(out[module], it)
	}
	return out, nil
}

// Items is what the modules on this machine declare; none when the mesh composed no data file — an
// older controller, which composes none.
func (b *Backups) Items() (map[string][]Item, error) {
	if b.Where.Data == "" {
		return map[string][]Item{}, nil
	}
	raw, err := os.ReadFile(b.Where.Data)
	if errors.Is(err, os.ErrNotExist) {
		return map[string][]Item{}, nil
	}
	if err != nil {
		return nil, err
	}
	return parseItems(string(raw))
}

func (b *Backups) measuredFile() string { return filepath.Join(b.Where.State, "measured.json") }

// Measurements is the newest measurement of each item, by module and item.
func (b *Backups) Measurements() map[string]map[string]Measured {
	out := map[string]map[string]Measured{}
	if raw, err := os.ReadFile(b.measuredFile()); err == nil {
		_ = json.Unmarshal(raw, &out)
	}
	return out
}

var measuring sync.Mutex

// measureCommand sums the files under a path and finds its newest change, in one walk, as root: a
// store's directory is often its own user's alone. One line out: "<bytes> <epoch seconds>".
func measureCommand(path string) string {
	return "find '" + path + "' -xdev -printf '%y %s %T@\\n' 2>/dev/null | " +
		"awk 'BEGIN{s=0;m=0} {if ($1==\"f\") s+=$2; if ($3>m) m=$3} END {printf \"%d %.0f\\n\", s, m}'"
}

// measure is one item, measured now.
func (b *Backups) measure(ctx context.Context, it Item) Measured {
	at := b.Now().UTC()
	m := Measured{MeasuredAt: &at}
	if !b.exists(ctx, it.Path) {
		m.Error = it.Path + " does not exist"
		zero := int64(0)
		m.SizeBytes = &zero
		return m
	}
	out, err := b.Run(ctx, "sh", "-c", measureCommand(it.Path))
	if err != nil {
		m.Error = err.Error()
		return m
	}
	f := strings.Fields(out)
	if len(f) != 2 {
		m.Error = "the measurement said " + strings.TrimSpace(out)
		return m
	}
	size, err1 := strconv.ParseInt(f[0], 10, 64)
	epoch, err2 := strconv.ParseInt(f[1], 10, 64)
	if err1 != nil || err2 != nil {
		m.Error = "the measurement said " + strings.TrimSpace(out)
		return m
	}
	m.SizeBytes = &size
	if epoch > 0 {
		w := time.Unix(epoch, 0).UTC()
		m.LastWrite = &w
	}
	m.Redundancy = b.redundancyOf(ctx, it.Path)
	return m
}

// redundancyOf is the redundant storage a path is on, read as root: a ZFS pool's health and its own
// verdict, an md array's members, a btrfs filesystem's error counters. Nil for storage with no
// redundancy this holder knows how to read — which, for an item said to be protected by redundancy, the
// controller says is no protection at all.
func (b *Backups) redundancyOf(ctx context.Context, path string) *Redundancy {
	out, err := b.Run(ctx, "findmnt", "-no", "SOURCE,FSTYPE", "--target", path)
	if err != nil {
		return nil
	}
	f := strings.Fields(out)
	if len(f) < 2 {
		return nil
	}
	source, fstype := f[0], f[1]
	yes, no := true, false
	switch {
	case fstype == "zfs":
		pool, _, _ := strings.Cut(source, "/")
		r := &Redundancy{Kind: "zfs", Where: pool}
		health, err := b.Run(ctx, "zpool", "list", "-H", "-o", "health", pool)
		if err != nil {
			r.Said = "zpool list: " + err.Error()
			return r
		}
		status, err := b.Run(ctx, "zpool", "status", "-x", pool)
		if err != nil {
			r.Said = "zpool status: " + err.Error()
			return r
		}
		health, status = strings.TrimSpace(health), strings.TrimSpace(status)
		r.Said = "health " + health + "; " + firstLineOf(status)
		if health == "ONLINE" && strings.Contains(status, "is healthy") {
			r.Healthy = &yes
		} else {
			r.Healthy = &no
			r.Said = "health " + health + "; " + oneLineOf(status)
		}
		return r
	case strings.HasPrefix(source, "/dev/md"):
		device := strings.TrimPrefix(source, "/dev/")
		r := &Redundancy{Kind: "md", Where: device}
		mdstat, err := b.Run(ctx, "cat", "/proc/mdstat")
		if err != nil {
			r.Said = err.Error()
			return r
		}
		healthy, said, found := mdHealth(mdstat, device)
		if !found {
			r.Said = device + " is not in /proc/mdstat"
			return r
		}
		r.Said = said
		if healthy {
			r.Healthy = &yes
		} else {
			r.Healthy = &no
		}
		return r
	case fstype == "btrfs":
		r := &Redundancy{Kind: "btrfs", Where: source}
		stats, err := b.Run(ctx, "btrfs", "device", "stats", "--check", path)
		if err != nil {
			r.Healthy, r.Said = &no, "device errors: "+oneLineOf(err.Error())
			return r
		}
		r.Healthy, r.Said = &yes, firstLineOf(stats)
		return r
	}
	return nil
}

var mdMembers = regexp.MustCompile(`\[([U_]+)\]`)

// mdHealth reads one array's block of /proc/mdstat: healthy when every member is up and it is not
// recovering.
func mdHealth(mdstat, device string) (bool, string, bool) {
	lines := strings.Split(mdstat, "\n")
	for i, l := range lines {
		if !strings.HasPrefix(l, device+" ") {
			continue
		}
		block := l
		for j := i + 1; j < len(lines) && strings.HasPrefix(lines[j], " "); j++ {
			block += " " + strings.TrimSpace(lines[j])
		}
		members := mdMembers.FindStringSubmatch(block)
		healthy := members != nil && !strings.Contains(members[1], "_") && !strings.Contains(block, "recovery")
		return healthy, oneLineOf(block), true
	}
	return false, "", false
}

func firstLineOf(s string) string {
	line, _, _ := strings.Cut(strings.TrimSpace(s), "\n")
	return line
}

func oneLineOf(s string) string { return strings.Join(strings.Fields(s), " ") }

// MeasureAll measures every item that is not a cache, one at a time, and keeps what it found.
func (b *Backups) MeasureAll(ctx context.Context) error {
	measuring.Lock()
	defer measuring.Unlock()
	items, err := b.Items()
	if err != nil {
		return err
	}
	found := map[string]map[string]Measured{}
	for module, its := range items {
		for _, it := range its {
			if it.Class == classCache {
				continue
			}
			if found[module] == nil {
				found[module] = map[string]Measured{}
			}
			found[module][it.ID] = b.measure(ctx, it)
		}
	}
	raw, _ := json.MarshalIndent(found, "", "  ")
	return os.WriteFile(b.measuredFile(), append(raw, '\n'), 0o600)
}

func (b *Backups) goodFile() string { return filepath.Join(b.Where.State, "good.json") }

// Good is each module's newest good night: what a night that failed since does not erase.
func (b *Backups) Good() map[string]time.Time {
	out := map[string]time.Time{}
	if raw, err := os.ReadFile(b.goodFile()); err == nil {
		_ = json.Unmarshal(raw, &out)
	}
	// A night recorded good before this file existed counts.
	for module, n := range b.Nights() {
		if n.OK && n.At.After(out[module]) {
			out[module] = n.At
		}
	}
	return out
}

func (b *Backups) recordGood(module string, at time.Time) {
	good := b.Good()
	good[module] = at
	raw, _ := json.MarshalIndent(good, "", "  ")
	if err := os.WriteFile(b.goodFile(), append(raw, '\n'), 0o600); err != nil {
		b.Say("recording %s's good night failed: %v", module, err)
	}
}

// itemReports is every item of one module, with its newest measurement and its newest good backup: the
// module's newest good night, where that night keeps the path that covers the item.
func itemReports(its []Item, measured map[string]Measured, paths []string, good time.Time) []ItemReport {
	out := []ItemReport{}
	for _, it := range its {
		r := ItemReport{Item: it, Measured: measured[it.ID]}
		if it.CoveredBy != "" && !good.IsZero() {
			for _, p := range paths {
				if p == it.CoveredBy {
					g := good
					r.LastBackup = &g
				}
			}
		}
		out = append(out, r)
	}
	return out
}

// ---- deleting a retired item -------------------------------------------------------------------

// Deletion is how one deletion went, by path.
type Deletion struct {
	Module   string    `json:"module"`
	Item     string    `json:"item"`
	Started  time.Time `json:"started"`
	Running  bool      `json:"running"`
	Done     bool      `json:"done"`
	OK       bool      `json:"ok"`
	Snapshot string    `json:"snapshot,omitempty"`
	Error    string    `json:"error,omitempty"`
	By       string    `json:"by,omitempty"`
	Why      string    `json:"why,omitempty"`
}

func (b *Backups) deletionsFile() string { return filepath.Join(b.Where.State, "deleted.json") }

var deletionsMu sync.Mutex

func (b *Backups) deletions() map[string]Deletion {
	out := map[string]Deletion{}
	if raw, err := os.ReadFile(b.deletionsFile()); err == nil {
		_ = json.Unmarshal(raw, &out)
	}
	return out
}

func (b *Backups) keepDeletion(path string, d Deletion) {
	deletionsMu.Lock()
	defer deletionsMu.Unlock()
	all := b.deletions()
	all[path] = d
	raw, _ := json.MarshalIndent(all, "", "  ")
	if err := os.WriteFile(b.deletionsFile(), append(raw, '\n'), 0o600); err != nil {
		b.Say("recording the deletion of %s failed: %v", path, err)
	}
}

// refuseDeleting says why a path may not be deleted: not absolute, too near the root, or declared now
// — by any module's item or backup line on this machine, itself, inside one, or holding one.
func (b *Backups) refuseDeleting(path string) error {
	clean := filepath.Clean(path)
	if !safePath.MatchString(path) || clean != strings.TrimRight(path, "/") {
		return fmt.Errorf("%q is not a path this holder deletes", path)
	}
	if strings.Count(clean, "/") < 2 {
		return fmt.Errorf("%s is too near the root to be a module's data", clean)
	}
	near := func(declared string) bool {
		d := filepath.Clean(declared)
		return d == clean || strings.HasPrefix(d, clean+"/") || strings.HasPrefix(clean, d+"/")
	}
	items, err := b.Items()
	if err != nil {
		return fmt.Errorf("what is declared here cannot be read, so nothing is deleted: %v", err)
	}
	for module, its := range items {
		for _, it := range its {
			if near(it.Path) {
				return fmt.Errorf("%s is declared now — %s's %s at %s — so it is not retired; nothing is deleted",
					clean, module, it.ID, it.Path)
			}
		}
	}
	declared, err := b.Declared()
	if err != nil && !errors.Is(err, os.ErrNotExist) {
		return fmt.Errorf("what is backed up here cannot be read, so nothing is deleted: %v", err)
	}
	for _, d := range declared {
		for _, p := range d.Paths {
			if near(p) {
				return fmt.Errorf("%s is backed up now as %s's %s, so it is not retired; nothing is deleted", clean, d.Module, p)
			}
		}
	}
	if clean == filepath.Clean(b.Where.Repository) || strings.HasPrefix(b.Where.Repository, clean+"/") ||
		clean == filepath.Clean(b.Where.State) {
		return fmt.Errorf("%s holds this machine's backups; nothing is deleted", clean)
	}
	return nil
}

// DeleteRetired starts deleting one retired item: a last restore point of it, tagged as retired, then
// its removal — never the removal without the restore point. Answers at once; DeletedOutcome follows
// it. A path whose deletion already finished answers how it went.
func (b *Backups) DeleteRetired(ctx context.Context, module, item, path, confirm, by, why string) (Deletion, error) {
	if module == "" || item == "" || path == "" {
		return Deletion{}, errors.New("module, item and path are required")
	}
	if confirm != item {
		return Deletion{}, fmt.Errorf("confirm is the item's name again (%s), to say this is meant", item)
	}
	if strings.TrimSpace(why) == "" {
		return Deletion{}, errors.New("why is required: a deletion is a person's decision, and says why")
	}
	if d, ok := b.deletions()[path]; ok && (d.Running || (d.Done && d.OK)) {
		return d, nil
	}
	if err := b.refuseDeleting(path); err != nil {
		return Deletion{}, err
	}
	if !b.exists(ctx, path) {
		d := Deletion{Module: module, Item: item, Started: b.Now().UTC(), Done: true, OK: true,
			Error: path + " was already gone", By: by, Why: why}
		b.keepDeletion(path, d)
		return d, nil
	}
	d := Deletion{Module: module, Item: item, Started: b.Now().UTC(), Running: true, By: by, Why: why}
	b.keepDeletion(path, d)
	go b.deleteNow(context.WithoutCancel(ctx), path, d)
	return d, nil
}

func (b *Backups) deleteNow(ctx context.Context, path string, d Deletion) {
	b.mu.Lock()
	defer b.mu.Unlock()
	finish := func(err error) {
		d.Running, d.Done = false, true
		if err != nil {
			d.Error = err.Error()
			b.Say("%s's retired %s at %s was NOT deleted: %v", d.Module, d.Item, path, err)
		} else {
			d.OK = true
			b.Say("%s's retired %s at %s DELETED by %s: %s; its last restore point is %s", d.Module, d.Item, path,
				orSomebody(d.By), d.Why, d.Snapshot)
		}
		b.keepDeletion(path, d)
	}
	if err := b.ensureRepository(ctx); err != nil {
		finish(fmt.Errorf("no restore point could be taken first: %w", err))
		return
	}
	out, err := b.restic(ctx, "backup", "--json", "--tag", tagOf(d.Module), "--tag", "retired="+d.Item, path)
	if err != nil {
		finish(fmt.Errorf("the last restore point could not be taken, so nothing was deleted: %w", err))
		return
	}
	d.Snapshot = snapshotOf(out)
	if d.Snapshot == "" {
		finish(errors.New("restic took the last restore point and named none, so nothing was deleted"))
		return
	}
	if _, err := b.Run(ctx, "rm", "-rf", "--one-file-system", "--", path); err != nil {
		finish(err)
		return
	}
	finish(nil)
}

// DeletedOutcome is how the deletion of a path went.
func (b *Backups) DeletedOutcome(path string) (Deletion, error) {
	d, ok := b.deletions()[path]
	if !ok {
		return Deletion{}, fmt.Errorf("no deletion of %s was asked of this holder", path)
	}
	return d, nil
}

func orSomebody(by string) string {
	if by == "" {
		return "somebody"
	}
	return by
}