package main import ( "context" "fmt" "regexp" "strconv" "strings" "time" ) // ScrubWarnAfter is how long after the last completed scrub the module warns: a month and a few days, so // a monthly scrub that ran late is not yet a warning. const ScrubWarnAfter = 35 * 24 * time.Hour // HistoryTimeout bounds reading a pool's history, which grows with the pool's life. const HistoryTimeout = 60 * time.Second // HistoryArgv is one pool's history with ZFS's internal events, where each scan's start and end are. func HistoryArgv(pool string) []string { return []string{"history", "-i", pool} } // Pass is one scan the history records: when it started and how it ended. type Pass struct { Kind string `json:"kind"` Started *time.Time `json:"started,omitempty"` Ended *time.Time `json:"ended,omitempty"` // Outcome is finished or canceled. Outcome string `json:"outcome"` Errors *int `json:"errors,omitempty"` } var historyScan = regexp.MustCompile(`^(\d{4}-\d{2}-\d{2}\.\d{2}:\d{2}:\d{2}) \[txg:\d+\] scan (setup|done|cancelled|canceled|aborted)\b(.*)$`) var historyFunc = regexp.MustCompile(`\bfunc=(\d+)`) var historyErrors = regexp.MustCompile(`\berrors=(\d+)`) // ParseHistory reads `zpool history -i` for the last finished scrub and the last finished resilver. ZFS // writes `scan setup … func=1` for a scrub and `func=2` for a resilver, then `scan done … errors=N` or // `scan cancelled`. func ParseHistory(out string) (scrub, resilver *Pass) { var open *Pass for _, line := range lines(out) { m := historyScan.FindStringSubmatch(strings.TrimSpace(line)) if m == nil { continue } t, err := time.ParseInLocation("2006-01-02.15:04:05", m[1], time.Local) if err != nil { continue } switch m[2] { case "setup": kind := "scan" if f := historyFunc.FindStringSubmatch(m[3]); f != nil { switch f[1] { case "1": kind = "scrub" case "2": kind = "resilver" } } open = &Pass{Kind: kind, Started: &t} default: if open == nil { continue } open.Ended = &t open.Outcome = "finished" if m[2] != "done" { open.Outcome = "canceled" } if e := historyErrors.FindStringSubmatch(m[3]); e != nil { n, _ := strconv.Atoi(e[1]) open.Errors = &n } if open.Outcome == "finished" { switch open.Kind { case "scrub": scrub = open case "resilver": resilver = open } } open = nil } } return scrub, resilver } // ScrubRow is one pool's scans. type ScrubRow struct { Pool string `json:"pool"` // Scans are the pool's scan lines as status gives them: its scrub or error scrub, then its resilver. Scans []Scan `json:"scans"` // LastScrub and LastResilver are the last finished of each, from status, or from the history when it // was read and status no longer shows it. LastScrub *Pass `json:"last_scrub,omitempty"` LastResilver *Pass `json:"last_resilver,omitempty"` DaysSince *float64 `json:"days_since_last_scrub,omitempty"` Warning string `json:"warning,omitempty"` Note string `json:"note,omitempty"` Said []string `json:"said"` } // ScrubAnswer is what zfs_scrub answers. type ScrubAnswer struct { Applicable bool `json:"applicable"` Pools []ScrubRow `json:"pools"` Schedule *Schedule `json:"schedule,omitempty"` Said []string `json:"said"` } // LastScrubEnded is when the pool's last scrub finished, as status shows it; nil when status shows no // finished scrub. func LastScrubEnded(p PoolStatus) *time.Time { if s := p.Scrub(); s != nil && s.State == "finished" { return s.Ended } return nil } // ScrubLate is how long after its last finished scrub a pool is late: its schedule's cadence and margin, // or ScrubWarnAfter where the module has no schedule for it. func ScrubLate(pool string, s *Schedule) (time.Duration, string) { if s != nil && s.Pool == pool { return cadenceAllows[s.Cadence], "its " + s.Cadence + " schedule allows" } return ScrubWarnAfter, "the module's line for a pool without a schedule" } // Scrubs are every pool's scans, its schedule and when it fires next. With history, each pool's history // is read too (as root), for the scrub and the resilver status no longer shows. func (m *Machine) Scrubs(ctx context.Context, history bool) (any, error) { st, err := m.Status(ctx) if err != nil { return notHere(err) } now := m.Now() a := ScrubAnswer{Applicable: true, Pools: []ScrubRow{}, Said: []string{}} sched, serr := m.ScheduleNow(ctx) switch { case serr != nil: a.Said = append(a.Said, "The scrub schedule's settings are not valid: "+serr.Error()+".") case sched == nil: a.Said = append(a.Said, "The module has no scrub schedule on this machine (no "+ScrubConfig+").") default: a.Schedule = sched line := fmt.Sprintf("%s scrub of %s, %s at %s (monthly: the first %s of the month), decided by the run %s fires "+ "every quarter of an hour (%s, %s)", sched.Cadence, sched.Pool, sched.Day, sched.Time, sched.Day, sched.Timer, orUnknown(sched.TimerState), orUnknown(sched.Enabled)) if sched.Cadence == "weekly" { line = fmt.Sprintf("weekly scrub of %s, %s at %s, decided by the run %s fires every quarter of an hour (%s, %s)", sched.Pool, sched.Day, sched.Time, sched.Timer, orUnknown(sched.TimerState), orUnknown(sched.Enabled)) } if sched.Next != nil { line += "; next scheduled scrub of " + sched.Pool + ": " + sched.Next.Local().Format("2006-01-02 15:04") } if sched.LastResult != "" { line += "; last run: " + sched.LastResult } a.Said = append(a.Said, line) if sched.Note != "" { a.Said = append(a.Said, sched.Note) } } for _, p := range st { r := ScrubRow{Pool: p.Name, Scans: p.Scans, Said: []string{}} if ended := LastScrubEnded(p); ended != nil { sc := p.Scrub() r.LastScrub = &Pass{Kind: "scrub", Started: sc.Started, Ended: ended, Outcome: "finished", Errors: sc.Errors} } for _, rs := range p.Resilvers() { if rs.State == "finished" && rs.Ended != nil && (r.LastResilver == nil || rs.Ended.After(*r.LastResilver.Ended)) { r.LastResilver = &Pass{Kind: "resilver", Started: rs.Started, Ended: rs.Ended, Outcome: "finished", Errors: rs.Errors} } } if history { hctx, cancel := context.WithTimeout(ctx, HistoryTimeout) out, err := m.AsRoot(hctx, "zpool", HistoryArgv(p.Name)...) cancel() if err != nil { r.Note = "the history could not be read: " + err.Error() } else { scrub, resilver := ParseHistory(out.Out) if scrub != nil && (r.LastScrub == nil || scrub.Ended.After(*r.LastScrub.Ended)) { r.LastScrub = scrub } if resilver != nil && (r.LastResilver == nil || resilver.Ended.After(*r.LastResilver.Ended)) { r.LastResilver = resilver } } } else if r.LastScrub == nil { r.Note = "status shows no finished scrub of this pool; ask with history=true for the last one the pool's history records" } late, by := ScrubLate(p.Name, sched) if r.LastScrub != nil && r.LastScrub.Ended != nil { d := round1(now.Sub(*r.LastScrub.Ended).Hours() / 24) r.DaysSince = &d if now.Sub(*r.LastScrub.Ended) > late { r.Warning = fmt.Sprintf("the last scrub of %s finished %.0f days ago, more than %s (%d days)", p.Name, d, by, int(late.Hours()/24)) } } else if history { r.Warning = fmt.Sprintf("no finished scrub of %s is on record", p.Name) } for _, sc := range p.Scans { r.Said = append(r.Said, p.Name+": "+scanSaid(sc)) } if len(p.Scans) == 0 { r.Said = append(r.Said, p.Name+": no scan was ever asked") } if r.DaysSince != nil { r.Said = append(r.Said, fmt.Sprintf("%s: last finished scrub %.1f days ago", p.Name, *r.DaysSince)) } if r.Warning != "" { r.Said = append(r.Said, p.Name+": WARNING: "+r.Warning) } if r.Note != "" { r.Said = append(r.Said, p.Name+": "+r.Note) } a.Pools = append(a.Pools, r) a.Said = append(a.Said, r.Said...) } return a, nil } // scanSaid says one scan in a line: a running one with its progress and the time to go. func scanSaid(s Scan) string { what := s.Kind if s.Vdev != "" { what += " of " + s.Vdev } switch s.State { case "running", "paused": line := fmt.Sprintf("a %s is %s", what, s.State) if s.Started != nil { line += " (started " + s.Started.Local().Format("2006-01-02 15:04") + ")" } if s.Done != nil { line += fmt.Sprintf(", %.2f %% done", *s.Done) } switch { case s.ToGo != "": line += ", " + s.ToGo + " to go" case s.NoEstimate: line += ", no estimate of the time to go yet" } return line case "finished": line := fmt.Sprintf("the last %s finished %s after %s, %s", what, stamp(s.Ended), s.Took, s.Repaired) if s.Kind == "scrub" { line += " repaired" } else if s.Kind == "resilver" { line += " resilvered" } if s.Errors != nil { line += fmt.Sprintf(", %d errors", *s.Errors) } return line case "canceled": return fmt.Sprintf("the last %s was canceled %s", what, stamp(s.Ended)) case "none": return "no scan was ever asked" } return "scan: " + firstLine(s.Said) } func orUnknown(s string) string { if s == "" { return "not known" } return s } func stamp(t *time.Time) string { if t == nil { return "at a time status did not say" } return t.Local().Format("2006-01-02 15:04") } func deref(n *int) int { if n == nil { return 0 } return *n }