package main import ( "context" "fmt" "regexp" "strconv" "strings" "time" ) // ScrubWarnAfter is how long after the last completed scrub the module warns: a month and a few days, so // a monthly scrub that ran late is not yet a warning. const ScrubWarnAfter = 35 * 24 * time.Hour // HistoryTimeout bounds reading a pool's history, which grows with the pool's life. const HistoryTimeout = 60 * time.Second // HistoryArgv is one pool's history with ZFS's internal events, where each scan's start and end are. func HistoryArgv(pool string) []string { return []string{"history", "-i", pool} } // Pass is one scan the history records: when it started and how it ended. type Pass struct { Kind string `json:"kind"` Started *time.Time `json:"started,omitempty"` Ended *time.Time `json:"ended,omitempty"` // Outcome is finished or canceled. Outcome string `json:"outcome"` Errors *int `json:"errors,omitempty"` } var historyScan = regexp.MustCompile(`^(\d{4}-\d{2}-\d{2}\.\d{2}:\d{2}:\d{2}) \[txg:\d+\] scan (setup|done|cancelled|canceled|aborted)\b(.*)$`) var historyFunc = regexp.MustCompile(`\bfunc=(\d+)`) var historyErrors = regexp.MustCompile(`\berrors=(\d+)`) // ParseHistory reads `zpool history -i` for the last finished scrub and the last finished resilver. ZFS // writes `scan setup … func=1` for a scrub and `func=2` for a resilver, then `scan done … errors=N` or // `scan cancelled`. func ParseHistory(out string) (scrub, resilver *Pass) { var open *Pass for _, line := range lines(out) { m := historyScan.FindStringSubmatch(strings.TrimSpace(line)) if m == nil { continue } t, err := time.ParseInLocation("2006-01-02.15:04:05", m[1], time.Local) if err != nil { continue } switch m[2] { case "setup": kind := "scan" if f := historyFunc.FindStringSubmatch(m[3]); f != nil { switch f[1] { case "1": kind = "scrub" case "2": kind = "resilver" } } open = &Pass{Kind: kind, Started: &t} default: if open == nil { continue } open.Ended = &t open.Outcome = "finished" if m[2] != "done" { open.Outcome = "canceled" } if e := historyErrors.FindStringSubmatch(m[3]); e != nil { n, _ := strconv.Atoi(e[1]) open.Errors = &n } if open.Outcome == "finished" { switch open.Kind { case "scrub": scrub = open case "resilver": resilver = open } } open = nil } } return scrub, resilver } // ScrubRow is one pool's scans. type ScrubRow struct { Pool string `json:"pool"` // Last is the pool's last scan as status says it: ZFS keeps only the latest, a scrub's or a resilver's. Last Scan `json:"last_scan"` // LastScrub and LastResilver are the last finished of each, from the history when it was read. LastScrub *Pass `json:"last_scrub,omitempty"` LastResilver *Pass `json:"last_resilver,omitempty"` DaysSince *float64 `json:"days_since_last_scrub,omitempty"` Warning string `json:"warning,omitempty"` Note string `json:"note,omitempty"` Said string `json:"said"` } // ScrubAnswer is what zfs_scrub answers. type ScrubAnswer struct { Applicable bool `json:"applicable"` Pools []ScrubRow `json:"pools"` Said []string `json:"said"` } // LastScrubEnded is when the pool's last scrub finished, as status shows it; nil when status's last scan // is not a finished scrub. func LastScrubEnded(s Scan) *time.Time { if s.Kind == "scrub" && s.State == "finished" { return s.Ended } return nil } // Scrubs are every pool's last scrub and resilver. With history, each pool's history is read too (as // root), for the scan status no longer shows. func (m *Machine) Scrubs(ctx context.Context, history bool) (any, error) { st, err := m.Status(ctx) if err != nil { return notHere(err) } now := m.Now() a := ScrubAnswer{Applicable: true, Pools: []ScrubRow{}, Said: []string{}} for _, p := range st { r := ScrubRow{Pool: p.Name, Last: p.Scan} ended := LastScrubEnded(p.Scan) if p.Scan.Kind == "resilver" && p.Scan.State == "finished" && p.Scan.Ended != nil { r.LastResilver = &Pass{Kind: "resilver", Started: p.Scan.Started, Ended: p.Scan.Ended, Outcome: "finished", Errors: p.Scan.Errors} } if ended != nil { r.LastScrub = &Pass{Kind: "scrub", Started: p.Scan.Started, Ended: ended, Outcome: "finished", Errors: p.Scan.Errors} } if history { hctx, cancel := context.WithTimeout(ctx, HistoryTimeout) out, err := m.AsRoot(hctx, "zpool", HistoryArgv(p.Name)...) cancel() if err != nil { r.Note = "the history could not be read: " + err.Error() } else { scrub, resilver := ParseHistory(out.Out) if scrub != nil && (r.LastScrub == nil || scrub.Ended.After(*r.LastScrub.Ended)) { r.LastScrub = scrub } if resilver != nil && (r.LastResilver == nil || resilver.Ended.After(*r.LastResilver.Ended)) { r.LastResilver = resilver } } } else if r.LastScrub == nil { r.Note = "status shows only the last scan, which is not a finished scrub; ask with history=true for the last scrub the pool's history records" } if r.LastScrub != nil && r.LastScrub.Ended != nil { d := round1(now.Sub(*r.LastScrub.Ended).Hours() / 24) r.DaysSince = &d if now.Sub(*r.LastScrub.Ended) > ScrubWarnAfter { r.Warning = fmt.Sprintf("the last scrub of %s finished %.0f days ago, more than %d", p.Name, d, int(ScrubWarnAfter.Hours()/24)) } } else if history { r.Warning = fmt.Sprintf("no finished scrub of %s is on record", p.Name) } r.Said = scrubSaid(r) a.Pools = append(a.Pools, r) a.Said = append(a.Said, r.Said) } return a, nil } func scrubSaid(r ScrubRow) string { s := r.Last var line string switch s.State { case "none": line = fmt.Sprintf("%s: no scan was ever asked", r.Pool) case "running", "paused": line = fmt.Sprintf("%s: a %s is %s", r.Pool, s.Kind, s.State) if s.Started != nil { line += " since " + s.Started.Local().Format("2006-01-02 15:04") } if s.Done != nil { line += fmt.Sprintf(", %.2f %% done", *s.Done) } if s.ToGo != "" { line += ", " + s.ToGo + " to go" } case "finished": line = fmt.Sprintf("%s: the last scan was a %s, finished %s after %s, %s repaired, %d errors", r.Pool, s.Kind, stamp(s.Ended), s.Took, s.Repaired, deref(s.Errors)) case "canceled": line = fmt.Sprintf("%s: the last %s was canceled %s", r.Pool, s.Kind, stamp(s.Ended)) default: line = fmt.Sprintf("%s: scan: %s", r.Pool, firstLine(s.Said)) } if r.DaysSince != nil { line += fmt.Sprintf("; last finished scrub %.1f days ago", *r.DaysSince) } if r.Warning != "" { line += " — WARNING: " + r.Warning } return line } func stamp(t *time.Time) string { if t == nil { return "at a time status did not say" } return t.Local().Format("2006-01-02 15:04") } func deref(n *int) int { if n == nil { return 0 } return *n }