mesh-catalog/modules/zfs/cmd/zfs-tools/scrub.go

package main

import (
	"context"
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"time"
)

// ScrubWarnAfter is how long after the last completed scrub the module warns: a month and a few days, so
// a monthly scrub that ran late is not yet a warning.
const ScrubWarnAfter = 35 * 24 * time.Hour

// HistoryTimeout bounds reading a pool's history, which grows with the pool's life.
const HistoryTimeout = 60 * time.Second

// HistoryArgv is one pool's history with ZFS's internal events, where each scan's start and end are.
func HistoryArgv(pool string) []string { return []string{"history", "-i", pool} }

// Pass is one scan the history records: when it started and how it ended.
type Pass struct {
	Kind    string     `json:"kind"`
	Started *time.Time `json:"started,omitempty"`
	Ended   *time.Time `json:"ended,omitempty"`
	// Outcome is finished or canceled.
	Outcome string `json:"outcome"`
	Errors  *int   `json:"errors,omitempty"`
}

var historyScan = regexp.MustCompile(`^(\d{4}-\d{2}-\d{2}\.\d{2}:\d{2}:\d{2}) \[txg:\d+\] scan (setup|done|cancelled|canceled|aborted)\b(.*)$`)
var historyFunc = regexp.MustCompile(`\bfunc=(\d+)`)
var historyErrors = regexp.MustCompile(`\berrors=(\d+)`)

// ParseHistory reads `zpool history -i` for the last finished scrub and the last finished resilver. ZFS
// writes `scan setup … func=1` for a scrub and `func=2` for a resilver, then `scan done … errors=N` or
// `scan cancelled`.
func ParseHistory(out string) (scrub, resilver *Pass) {
	var open *Pass
	for _, line := range lines(out) {
		m := historyScan.FindStringSubmatch(strings.TrimSpace(line))
		if m == nil {
			continue
		}
		t, err := time.ParseInLocation("2006-01-02.15:04:05", m[1], time.Local)
		if err != nil {
			continue
		}
		switch m[2] {
		case "setup":
			kind := "scan"
			if f := historyFunc.FindStringSubmatch(m[3]); f != nil {
				switch f[1] {
				case "1":
					kind = "scrub"
				case "2":
					kind = "resilver"
				}
			}
			open = &Pass{Kind: kind, Started: &t}
		default:
			if open == nil {
				continue
			}
			open.Ended = &t
			open.Outcome = "finished"
			if m[2] != "done" {
				open.Outcome = "canceled"
			}
			if e := historyErrors.FindStringSubmatch(m[3]); e != nil {
				n, _ := strconv.Atoi(e[1])
				open.Errors = &n
			}
			if open.Outcome == "finished" {
				switch open.Kind {
				case "scrub":
					scrub = open
				case "resilver":
					resilver = open
				}
			}
			open = nil
		}
	}
	return scrub, resilver
}

// ScrubRow is one pool's scans.
type ScrubRow struct {
	Pool string `json:"pool"`
	// Last is the pool's last scan as status says it: ZFS keeps only the latest, a scrub's or a resilver's.
	Last Scan `json:"last_scan"`
	// LastScrub and LastResilver are the last finished of each, from the history when it was read.
	LastScrub    *Pass    `json:"last_scrub,omitempty"`
	LastResilver *Pass    `json:"last_resilver,omitempty"`
	DaysSince    *float64 `json:"days_since_last_scrub,omitempty"`
	Warning      string   `json:"warning,omitempty"`
	Note         string   `json:"note,omitempty"`
	Said         string   `json:"said"`
}

// ScrubAnswer is what zfs_scrub answers.
type ScrubAnswer struct {
	Applicable bool       `json:"applicable"`
	Pools      []ScrubRow `json:"pools"`
	Said       []string   `json:"said"`
}

// LastScrubEnded is when the pool's last scrub finished, as status shows it; nil when status's last scan
// is not a finished scrub.
func LastScrubEnded(s Scan) *time.Time {
	if s.Kind == "scrub" && s.State == "finished" {
		return s.Ended
	}
	return nil
}

// Scrubs are every pool's last scrub and resilver. With history, each pool's history is read too (as
// root), for the scan status no longer shows.
func (m *Machine) Scrubs(ctx context.Context, history bool) (any, error) {
	st, err := m.Status(ctx)
	if err != nil {
		return notHere(err)
	}
	now := m.Now()
	a := ScrubAnswer{Applicable: true, Pools: []ScrubRow{}, Said: []string{}}
	for _, p := range st {
		r := ScrubRow{Pool: p.Name, Last: p.Scan}
		ended := LastScrubEnded(p.Scan)
		if p.Scan.Kind == "resilver" && p.Scan.State == "finished" && p.Scan.Ended != nil {
			r.LastResilver = &Pass{Kind: "resilver", Started: p.Scan.Started, Ended: p.Scan.Ended, Outcome: "finished", Errors: p.Scan.Errors}
		}
		if ended != nil {
			r.LastScrub = &Pass{Kind: "scrub", Started: p.Scan.Started, Ended: ended, Outcome: "finished", Errors: p.Scan.Errors}
		}
		if history {
			hctx, cancel := context.WithTimeout(ctx, HistoryTimeout)
			out, err := m.AsRoot(hctx, "zpool", HistoryArgv(p.Name)...)
			cancel()
			if err != nil {
				r.Note = "the history could not be read: " + err.Error()
			} else {
				scrub, resilver := ParseHistory(out.Out)
				if scrub != nil && (r.LastScrub == nil || scrub.Ended.After(*r.LastScrub.Ended)) {
					r.LastScrub = scrub
				}
				if resilver != nil && (r.LastResilver == nil || resilver.Ended.After(*r.LastResilver.Ended)) {
					r.LastResilver = resilver
				}
			}
		} else if r.LastScrub == nil {
			r.Note = "status shows only the last scan, which is not a finished scrub; ask with history=true for the last scrub the pool's history records"
		}
		if r.LastScrub != nil && r.LastScrub.Ended != nil {
			d := round1(now.Sub(*r.LastScrub.Ended).Hours() / 24)
			r.DaysSince = &d
			if now.Sub(*r.LastScrub.Ended) > ScrubWarnAfter {
				r.Warning = fmt.Sprintf("the last scrub of %s finished %.0f days ago, more than %d", p.Name, d, int(ScrubWarnAfter.Hours()/24))
			}
		} else if history {
			r.Warning = fmt.Sprintf("no finished scrub of %s is on record", p.Name)
		}
		r.Said = scrubSaid(r)
		a.Pools = append(a.Pools, r)
		a.Said = append(a.Said, r.Said)
	}
	return a, nil
}

func scrubSaid(r ScrubRow) string {
	s := r.Last
	var line string
	switch s.State {
	case "none":
		line = fmt.Sprintf("%s: no scan was ever asked", r.Pool)
	case "running", "paused":
		line = fmt.Sprintf("%s: a %s is %s", r.Pool, s.Kind, s.State)
		if s.Started != nil {
			line += " since " + s.Started.Local().Format("2006-01-02 15:04")
		}
		if s.Done != nil {
			line += fmt.Sprintf(", %.2f %% done", *s.Done)
		}
		if s.ToGo != "" {
			line += ", " + s.ToGo + " to go"
		}
	case "finished":
		line = fmt.Sprintf("%s: the last scan was a %s, finished %s after %s, %s repaired, %d errors", r.Pool, s.Kind,
			stamp(s.Ended), s.Took, s.Repaired, deref(s.Errors))
	case "canceled":
		line = fmt.Sprintf("%s: the last %s was canceled %s", r.Pool, s.Kind, stamp(s.Ended))
	default:
		line = fmt.Sprintf("%s: scan: %s", r.Pool, firstLine(s.Said))
	}
	if r.DaysSince != nil {
		line += fmt.Sprintf("; last finished scrub %.1f days ago", *r.DaysSince)
	}
	if r.Warning != "" {
		line += " — WARNING: " + r.Warning
	}
	return line
}

func stamp(t *time.Time) string {
	if t == nil {
		return "at a time status did not say"
	}
	return t.Local().Format("2006-01-02 15:04")
}

func deref(n *int) int {
	if n == nil {
		return 0
	}
	return *n
}