package main import ( "context" "encoding/json" "fmt" "os" "path/filepath" "regexp" "strconv" "strings" "syscall" "time" ) // The scrub schedule the module owns. // // The node-engine writes ConfigFile from the config values. The node-engine runs the module's scheduled // process, `zfs-tools run`, from its own timer (RunTimer) every quarter of an hour, and the run decides from the // config whether a scrub is due: the last scheduled time not after now (the slot) not yet served. Due, it starts // the pool's scrub and returns; it never pauses or stops one and never waits for one (run.go). The day and the // time are read and checked by the parse (config.go), so no setting value ever reaches a unit file. // // The mesh holds the distribution's own scrub timers for the pool stopped and disabled, so one schedule // scrubs it; the node-engine gives them back running and enabled when the module goes (novox/hq ADR 0269). // RunTimer and RunService are the node-engine's units for the scheduled run (the process's name). const ( RunTimer = "zfs-mesh-run.timer" RunService = "zfs-mesh-run.service" ) // ServedFile is where the scheduled run keeps the last slot it served, per pool: in a folder of root's // alone (0700), apart from the operator account's state, so nothing the account runs can steer the run's // writes (third review of mesh-catalog #147). The tools, which run as the account, cannot read it, and // judge a missed scrub from status instead. const ServedFile = "/var/lib/zfs-scrub/scrub-served.json" // MissedAfter is how long after its slot a scrub that has not started is said to be missed: the run // fires every quarter of an hour. const MissedAfter = time.Hour // Margins after a cadence before a last scrub is said to be late: a weekly run that waited for a busy // pool, a monthly run on the first given weekday of a month (28 or 35 days apart). var cadenceAllows = map[string]time.Duration{ "weekly": 9 * 24 * time.Hour, "monthly": 38 * 24 * time.Hour, } // Schedule is the pool's scrub schedule: what the settings say and what the service manager says of it. type Schedule struct { Pool string `json:"pool"` Cadence string `json:"cadence"` Day string `json:"day"` Time string `json:"time"` Timer string `json:"timer"` Service string `json:"service"` // TimerState and Enabled are the timer's ActiveState and UnitFileState. TimerState string `json:"timer_state,omitempty"` Enabled string `json:"enabled,omitempty"` // Next is the next scheduled scrub, from the settings; TimerNext when the timer next fires the run that // decides (every quarter of an hour). Next *time.Time `json:"next_run,omitempty"` TimerNext *time.Time `json:"timer_next,omitempty"` LastTrigger *time.Time `json:"last_triggered,omitempty"` // LastResult is the service's Result of its last run (success, exit-code, …), LastExit when it ended. LastResult string `json:"last_result,omitempty"` ServiceState string `json:"service_state,omitempty"` LastExit *time.Time `json:"last_run_ended,omitempty"` Note string `json:"note,omitempty"` } var ( poolName = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_.:-]*$`) weekday = regexp.MustCompile(`^(Mon|Tue|Wed|Thu|Fri|Sat|Sun)$`) clock = regexp.MustCompile(`^([01][0-9]|2[0-3]):[0-5][0-9]$`) ) // ScheduleOf is the scrub schedule the config holds: the pool, and the cadence, day and time. nil when the // values are not all set (the module then has no scrub schedule, §11: not managed until set). func ScheduleOf(c *Config) *Schedule { if c.Pool == "" || c.ScrubCadence == "" || c.ScrubDay == "" || c.ScrubTime == "" { return nil } return &Schedule{Pool: c.Pool, Cadence: c.ScrubCadence, Day: c.ScrubDay, Time: c.ScrubTime, Timer: RunTimer, Service: RunService} } // TrimScheduleOf is the trim schedule: nil while trim-schedule is not weekly or monthly, or its day or time is // not set. func TrimScheduleOf(c *Config) *Schedule { if c.Pool == "" || (c.TrimSchedule != "weekly" && c.TrimSchedule != "monthly") || c.TrimDay == "" || c.TrimTime == "" { return nil } return &Schedule{Pool: c.Pool, Cadence: c.TrimSchedule, Day: c.TrimDay, Time: c.TrimTime, Timer: RunTimer, Service: RunService} } var weekdays = map[string]time.Weekday{"Sun": time.Sunday, "Mon": time.Monday, "Tue": time.Tuesday, "Wed": time.Wednesday, "Thu": time.Thursday, "Fri": time.Friday, "Sat": time.Saturday} // at is the scheduled time on one day. func (s *Schedule) at(day time.Time) time.Time { h, _ := strconv.Atoi(s.Time[:2]) mi, _ := strconv.Atoi(s.Time[3:]) return time.Date(day.Year(), day.Month(), day.Day(), h, mi, 0, 0, day.Location()) } // scheduledOn says whether a scrub is scheduled on a day: its weekday, and for a monthly one the first // such day of the month. func (s *Schedule) scheduledOn(day time.Time) bool { return day.Weekday() == weekdays[s.Day] && (s.Cadence == "weekly" || day.Day() <= 7) } // Slot is the last scheduled time not after now. func (s *Schedule) Slot(now time.Time) time.Time { day := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) for i := 0; i < 45; i++ { if s.scheduledOn(day) && !s.at(day).After(now) { return s.at(day) } day = day.AddDate(0, 0, -1) } return time.Time{} } // NextSlot is the first scheduled time after now. func (s *Schedule) NextSlot(now time.Time) time.Time { day := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) for i := 0; i < 45; i++ { if s.scheduledOn(day) && s.at(day).After(now) { return s.at(day) } day = day.AddDate(0, 0, 1) } return time.Time{} } // readServed is the last slot served per pool; none when the file is not there or not readable. func (m *Machine) readServed(path string) map[string]int64 { out := map[string]int64{} b, err := m.ReadFile(path) if err != nil { return out } _ = json.Unmarshal(b, &out) return out } // writeServed keeps the served slots, never through a link: the folder must be a real folder of this // process's account alone (0700, made so when missing), and the file is written new beside it with // O_NOFOLLOW and O_EXCL, then renamed over the old one, which replaces a link rather than following it. func writeServed(path string, served map[string]int64) error { dir := filepath.Dir(path) if _, err := os.Lstat(dir); os.IsNotExist(err) { if err := os.Mkdir(dir, 0o700); err != nil { return err } } fi, err := os.Lstat(dir) if err != nil { return err } st, _ := fi.Sys().(*syscall.Stat_t) switch { case fi.Mode()&os.ModeSymlink != 0 || !fi.IsDir(): return fmt.Errorf("%s is not a folder of its own (a link or a file): the served slot is not written", dir) case fi.Mode().Perm() != 0o700: return fmt.Errorf("%s is %v, not 0700: the served slot is not written", dir, fi.Mode().Perm()) case st != nil && int(st.Uid) != os.Geteuid(): return fmt.Errorf("%s belongs to uid %d, not this process's %d: the served slot is not written", dir, st.Uid, os.Geteuid()) } if fi, err := os.Lstat(path); err == nil && !fi.Mode().IsRegular() { return fmt.Errorf("%s is not a plain file (a link or something else): the served slot is not written", path) } b, _ := json.Marshal(served) tmp := path + ".new" _ = os.Remove(tmp) f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_EXCL|syscall.O_NOFOLLOW, 0o600) if err != nil { return err } if _, err := f.Write(b); err != nil { f.Close() return err } if err := f.Close(); err != nil { return err } return os.Rename(tmp, path) } // startedSince says whether status shows the pool's scrub started at or after a time. func startedSince(p *PoolStatus, slot time.Time) bool { if p == nil { return false } sc := p.Scrub() return sc != nil && sc.Started != nil && !sc.Started.Before(slot) } // TimerShowArgv asks the service manager what a timer says: whether it runs and is enabled, when it // fires next and when it last did, as Unix times. func TimerShowArgv(unit string) []string { return []string{"show", "--timestamp=unix", "--property=ActiveState,UnitFileState,NextElapseUSecRealtime,LastTriggerUSec", "--", unit} } // ServiceShowArgv asks what the scrub service's last run was. func ServiceShowArgv(unit string) []string { return []string{"show", "--timestamp=unix", "--property=ActiveState,Result,ExecMainStartTimestamp,ExecMainExitTimestamp", "--", unit} } // parseShow reads `systemctl show`'s key=value lines. func parseShow(out string) map[string]string { v := map[string]string{} for _, l := range lines(out) { if k, val, ok := strings.Cut(l, "="); ok { v[k] = strings.TrimSpace(val) } } return v } // unixStamp reads `--timestamp=unix`: "@1791842400"; nil for "", "n/a" or 0. func unixStamp(s string) *time.Time { s = strings.TrimPrefix(strings.TrimSpace(s), "@") n, err := strconv.ParseInt(s, 10, 64) if err != nil || n == 0 { return nil } t := time.Unix(n, 0) return &t } // ScheduleNow is the schedule with what the service manager says of it; nil when the module has no // schedule on this machine (no config file, or no cadence, day and time set). func (m *Machine) ScheduleNow(ctx context.Context) (*Schedule, error) { raw, err := m.ReadFile(ConfigFile) if err != nil { return nil, nil } c, refused := ParseConfig(raw, m.memMiB()) if len(refused) > 0 { return nil, fmt.Errorf("%s refused %s %s", ConfigFile, refused[0].Key, refused[0].Reason) } s := ScheduleOf(c) if s == nil { return nil, nil } if out, err := m.ask(ctx, "systemctl", TimerShowArgv(s.Timer)...); err != nil { s.Note = "the service manager did not say what the timer is: " + err.Error() } else { v := parseShow(out) s.TimerState, s.Enabled = v["ActiveState"], v["UnitFileState"] s.TimerNext, s.LastTrigger = unixStamp(v["NextElapseUSecRealtime"]), unixStamp(v["LastTriggerUSec"]) } next := s.NextSlot(m.Now()) s.Next = &next if out, err := m.ask(ctx, "systemctl", ServiceShowArgv(s.Service)...); err == nil { v := parseShow(out) s.ServiceState, s.LastResult, s.LastExit = v["ActiveState"], v["Result"], unixStamp(v["ExecMainExitTimestamp"]) } return s, nil } // scheduleFindings are the schedule's warnings: a timer not waiting, a last run that failed with no scrub // finished since, a slot past by MissedAfter that no scrub served. func scheduleFindings(s *Schedule, p *PoolStatus, now time.Time, served map[string]int64) []string { var warn []string var scrubEnded *time.Time if p != nil { scrubEnded = LastScrubEnded(*p) } if slot := s.Slot(now); now.Sub(slot) > MissedAfter && served[s.Pool] < slot.Unix() && !startedSince(p, slot) { warn = append(warn, fmt.Sprintf("The scheduled scrub of %s due %s has not started.", s.Pool, slot.Format("2006-01-02 15:04"))) } if s.TimerState != "" && (s.TimerState != "active" || s.Enabled != "enabled") { warn = append(warn, fmt.Sprintf("The scrub schedule's timer %s is %s and %s, so no scheduled scrub of %s will start.", s.Timer, s.TimerState, s.Enabled, s.Pool)) } if s.LastResult != "" && s.LastResult != "success" && s.ServiceState != "activating" && (scrubEnded == nil || s.LastExit == nil || scrubEnded.Before(*s.LastExit)) { when := "at a time the service manager did not say" if s.LastExit != nil { when = s.LastExit.Local().Format("2006-01-02 15:04") } warn = append(warn, fmt.Sprintf("The last scheduled scrub of %s failed (%s), %s; its journal says why.", s.Pool, s.LastResult, when)) } return warn } // ScrubEvent is the event each scrub act is published as (module.json's emits). const ScrubEvent = "scrub.requested" // ScrubArgv is each act's command: fixed words, the pool after `--`. func ScrubArgv(act, pool string) []string { switch act { case "pause": return []string{"scrub", "-p", "--", pool} case "stop": return []string{"scrub", "-s", "--", pool} } return []string{"scrub", "--", pool} } // ScrubActAnswer is what zfs_scrub_start, zfs_scrub_pause and zfs_scrub_stop answer. type ScrubActAnswer struct { Done bool `json:"done"` Act string `json:"act"` Pool string `json:"pool"` Why string `json:"why"` Ran []string `json:"ran"` After string `json:"after"` Recorded bool `json:"recorded"` Record string `json:"record"` Said string `json:"said"` } // ScrubAct starts, pauses or stops a pool's scrub, and answers what status says afterwards. A scrub reads // every block and rewrites from redundancy only a block it finds damaged; it adds I/O load, and never // changes what a file holds. func (m *Machine) ScrubAct(ctx context.Context, act, pool, why string) (ScrubActAnswer, error) { a := ScrubActAnswer{Act: act, Pool: pool, Why: why} switch act { case "start", "pause", "stop": default: return a, fmt.Errorf("%q is not an act of the scrub: start, pause or stop", act) } switch { case why == "": return a, fmt.Errorf("say why: a scrub act is a person's, and the why is kept with it") case strings.ContainsAny(why, "\n\r") || len(why) > MaxWhy: return a, fmt.Errorf("say why in one line of at most %d characters", MaxWhy) } ps, err := m.Pools(ctx) if err != nil { return a, err } known := false for _, p := range ps { known = known || p.Name == pool } if !known { return a, fmt.Errorf("there is no pool %q on this machine; zfs_pools lists them", pool) } program, argv := m.Escalated("zpool", ScrubArgv(act, pool)...) a.Ran = append([]string{program}, argv...) _, runErr := m.AsRoot(ctx, "zpool", ScrubArgv(act, pool)...) _, timedOut := runErr.(errTimedOut) if st, err := m.Status(ctx); err == nil { if p := statusOf(st, pool); p != nil { a.After = p.ScansSaid() } } a.Done = runErr == nil switch { case timedOut: a.Said = fmt.Sprintf("zpool scrub did not answer within its time and was ended: outcome unknown; check zfs_scrub. Status now: %s.", a.After) case runErr != nil: a.Said = fmt.Sprintf("NOT DONE: %v. Status now: %s.", runErr, a.After) default: a.Said = fmt.Sprintf("Asked %s to %s its scrub. Status now: %s. Why: %s.", pool, act, a.After, strings.TrimSuffix(why, ".")) } if m.Emit == nil { a.Record = "this process publishes no events" } else if err := m.Emit(ScrubEvent, map[string]any{"pool": pool, "act": act, "why": why, "done": a.Done, "after": a.After, "at": m.Now().UTC().Format(time.RFC3339)}); err != nil { a.Record = err.Error() } else { a.Recorded, a.Record = true, "published as the event "+ScrubEvent } if !a.Recorded { a.Said += " NOT RECORDED on the bus: " + a.Record + "; the pool's own history (zpool history) has the command." } if runErr != nil { return a, fmt.Errorf("%s", a.Said) } return a, nil }