378 lines
14 KiB
Go
378 lines
14 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"syscall"
|
|
"time"
|
|
)
|
|
|
|
// The scrub schedule the module owns.
|
|
//
|
|
// The node-engine writes ConfigFile from the config values. The node-engine runs the module's scheduled
|
|
// process, `zfs-tools run`, from its own timer (RunTimer) every quarter of an hour, and the run decides from the
|
|
// config whether a scrub is due: the last scheduled time not after now (the slot) not yet served. Due, it starts
|
|
// the pool's scrub and returns; it never pauses or stops one and never waits for one (run.go). The day and the
|
|
// time are read and checked by the parse (config.go), so no setting value ever reaches a unit file.
|
|
//
|
|
// The mesh holds the distribution's own scrub timers for the pool stopped and disabled, so one schedule
|
|
// scrubs it; the node-engine gives them back running and enabled when the module goes (novox/hq ADR 0269).
|
|
|
|
// RunTimer and RunService are the node-engine's units for the scheduled run (the process's name).
|
|
const (
|
|
RunTimer = "zfs-mesh-run.timer"
|
|
RunService = "zfs-mesh-run.service"
|
|
)
|
|
|
|
// ServedFile is where the scheduled run keeps the last slot it served, per pool: in a folder of root's
|
|
// alone (0700), apart from the operator account's state, so nothing the account runs can steer the run's
|
|
// writes (third review of mesh-catalog #147). The tools, which run as the account, cannot read it, and
|
|
// judge a missed scrub from status instead.
|
|
const ServedFile = "/var/lib/zfs-scrub/scrub-served.json"
|
|
|
|
// MissedAfter is how long after its slot a scrub that has not started is said to be missed: the run
|
|
// fires every quarter of an hour.
|
|
const MissedAfter = time.Hour
|
|
|
|
// Margins after a cadence before a last scrub is said to be late: a weekly run that waited for a busy
|
|
// pool, a monthly run on the first given weekday of a month (28 or 35 days apart).
|
|
var cadenceAllows = map[string]time.Duration{
|
|
"weekly": 9 * 24 * time.Hour,
|
|
"monthly": 38 * 24 * time.Hour,
|
|
}
|
|
|
|
// Schedule is the pool's scrub schedule: what the settings say and what the service manager says of it.
|
|
type Schedule struct {
|
|
Pool string `json:"pool"`
|
|
Cadence string `json:"cadence"`
|
|
Day string `json:"day"`
|
|
Time string `json:"time"`
|
|
Timer string `json:"timer"`
|
|
Service string `json:"service"`
|
|
// TimerState and Enabled are the timer's ActiveState and UnitFileState.
|
|
TimerState string `json:"timer_state,omitempty"`
|
|
Enabled string `json:"enabled,omitempty"`
|
|
// Next is the next scheduled scrub, from the settings; TimerNext when the timer next fires the run that
|
|
// decides (every quarter of an hour).
|
|
Next *time.Time `json:"next_run,omitempty"`
|
|
TimerNext *time.Time `json:"timer_next,omitempty"`
|
|
LastTrigger *time.Time `json:"last_triggered,omitempty"`
|
|
// LastResult is the service's Result of its last run (success, exit-code, …), LastExit when it ended.
|
|
LastResult string `json:"last_result,omitempty"`
|
|
ServiceState string `json:"service_state,omitempty"`
|
|
LastExit *time.Time `json:"last_run_ended,omitempty"`
|
|
Note string `json:"note,omitempty"`
|
|
}
|
|
|
|
var (
|
|
poolName = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_.:-]*$`)
|
|
weekday = regexp.MustCompile(`^(Mon|Tue|Wed|Thu|Fri|Sat|Sun)$`)
|
|
clock = regexp.MustCompile(`^([01][0-9]|2[0-3]):[0-5][0-9]$`)
|
|
)
|
|
|
|
// ScheduleOf is the scrub schedule the config holds: the pool, and the cadence, day and time. nil when the
|
|
// values are not all set (the module then has no scrub schedule, §11: not managed until set).
|
|
func ScheduleOf(c *Config) *Schedule {
|
|
if c.Pool == "" || c.ScrubCadence == "" || c.ScrubDay == "" || c.ScrubTime == "" {
|
|
return nil
|
|
}
|
|
return &Schedule{Pool: c.Pool, Cadence: c.ScrubCadence, Day: c.ScrubDay, Time: c.ScrubTime, Timer: RunTimer, Service: RunService}
|
|
}
|
|
|
|
// TrimScheduleOf is the trim schedule: nil while trim-schedule is not weekly or monthly, or its day or time is
|
|
// not set.
|
|
func TrimScheduleOf(c *Config) *Schedule {
|
|
if c.Pool == "" || (c.TrimSchedule != "weekly" && c.TrimSchedule != "monthly") || c.TrimDay == "" || c.TrimTime == "" {
|
|
return nil
|
|
}
|
|
return &Schedule{Pool: c.Pool, Cadence: c.TrimSchedule, Day: c.TrimDay, Time: c.TrimTime, Timer: RunTimer, Service: RunService}
|
|
}
|
|
|
|
var weekdays = map[string]time.Weekday{"Sun": time.Sunday, "Mon": time.Monday, "Tue": time.Tuesday,
|
|
"Wed": time.Wednesday, "Thu": time.Thursday, "Fri": time.Friday, "Sat": time.Saturday}
|
|
|
|
// at is the scheduled time on one day.
|
|
func (s *Schedule) at(day time.Time) time.Time {
|
|
h, _ := strconv.Atoi(s.Time[:2])
|
|
mi, _ := strconv.Atoi(s.Time[3:])
|
|
return time.Date(day.Year(), day.Month(), day.Day(), h, mi, 0, 0, day.Location())
|
|
}
|
|
|
|
// scheduledOn says whether a scrub is scheduled on a day: its weekday, and for a monthly one the first
|
|
// such day of the month.
|
|
func (s *Schedule) scheduledOn(day time.Time) bool {
|
|
return day.Weekday() == weekdays[s.Day] && (s.Cadence == "weekly" || day.Day() <= 7)
|
|
}
|
|
|
|
// Slot is the last scheduled time not after now.
|
|
func (s *Schedule) Slot(now time.Time) time.Time {
|
|
day := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location())
|
|
for i := 0; i < 45; i++ {
|
|
if s.scheduledOn(day) && !s.at(day).After(now) {
|
|
return s.at(day)
|
|
}
|
|
day = day.AddDate(0, 0, -1)
|
|
}
|
|
return time.Time{}
|
|
}
|
|
|
|
// NextSlot is the first scheduled time after now.
|
|
func (s *Schedule) NextSlot(now time.Time) time.Time {
|
|
day := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location())
|
|
for i := 0; i < 45; i++ {
|
|
if s.scheduledOn(day) && s.at(day).After(now) {
|
|
return s.at(day)
|
|
}
|
|
day = day.AddDate(0, 0, 1)
|
|
}
|
|
return time.Time{}
|
|
}
|
|
|
|
// readServed is the last slot served per pool; none when the file is not there or not readable.
|
|
func (m *Machine) readServed(path string) map[string]int64 {
|
|
out := map[string]int64{}
|
|
b, err := m.ReadFile(path)
|
|
if err != nil {
|
|
return out
|
|
}
|
|
_ = json.Unmarshal(b, &out)
|
|
return out
|
|
}
|
|
|
|
// writeServed keeps the served slots, never through a link: the folder must be a real folder of this
|
|
// process's account alone (0700, made so when missing), and the file is written new beside it with
|
|
// O_NOFOLLOW and O_EXCL, then renamed over the old one, which replaces a link rather than following it.
|
|
func writeServed(path string, served map[string]int64) error {
|
|
dir := filepath.Dir(path)
|
|
if _, err := os.Lstat(dir); os.IsNotExist(err) {
|
|
if err := os.Mkdir(dir, 0o700); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
fi, err := os.Lstat(dir)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
st, _ := fi.Sys().(*syscall.Stat_t)
|
|
switch {
|
|
case fi.Mode()&os.ModeSymlink != 0 || !fi.IsDir():
|
|
return fmt.Errorf("%s is not a folder of its own (a link or a file): the served slot is not written", dir)
|
|
case fi.Mode().Perm() != 0o700:
|
|
return fmt.Errorf("%s is %v, not 0700: the served slot is not written", dir, fi.Mode().Perm())
|
|
case st != nil && int(st.Uid) != os.Geteuid():
|
|
return fmt.Errorf("%s belongs to uid %d, not this process's %d: the served slot is not written", dir, st.Uid, os.Geteuid())
|
|
}
|
|
if fi, err := os.Lstat(path); err == nil && !fi.Mode().IsRegular() {
|
|
return fmt.Errorf("%s is not a plain file (a link or something else): the served slot is not written", path)
|
|
}
|
|
b, _ := json.Marshal(served)
|
|
tmp := path + ".new"
|
|
_ = os.Remove(tmp)
|
|
f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_EXCL|syscall.O_NOFOLLOW, 0o600)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err := f.Write(b); err != nil {
|
|
f.Close()
|
|
return err
|
|
}
|
|
if err := f.Close(); err != nil {
|
|
return err
|
|
}
|
|
return os.Rename(tmp, path)
|
|
}
|
|
|
|
// startedSince says whether status shows the pool's scrub started at or after a time.
|
|
func startedSince(p *PoolStatus, slot time.Time) bool {
|
|
if p == nil {
|
|
return false
|
|
}
|
|
sc := p.Scrub()
|
|
return sc != nil && sc.Started != nil && !sc.Started.Before(slot)
|
|
}
|
|
|
|
// TimerShowArgv asks the service manager what a timer says: whether it runs and is enabled, when it
|
|
// fires next and when it last did, as Unix times.
|
|
func TimerShowArgv(unit string) []string {
|
|
return []string{"show", "--timestamp=unix", "--property=ActiveState,UnitFileState,NextElapseUSecRealtime,LastTriggerUSec", "--", unit}
|
|
}
|
|
|
|
// ServiceShowArgv asks what the scrub service's last run was.
|
|
func ServiceShowArgv(unit string) []string {
|
|
return []string{"show", "--timestamp=unix", "--property=ActiveState,Result,ExecMainStartTimestamp,ExecMainExitTimestamp", "--", unit}
|
|
}
|
|
|
|
// parseShow reads `systemctl show`'s key=value lines.
|
|
func parseShow(out string) map[string]string {
|
|
v := map[string]string{}
|
|
for _, l := range lines(out) {
|
|
if k, val, ok := strings.Cut(l, "="); ok {
|
|
v[k] = strings.TrimSpace(val)
|
|
}
|
|
}
|
|
return v
|
|
}
|
|
|
|
// unixStamp reads `--timestamp=unix`: "@1791842400"; nil for "", "n/a" or 0.
|
|
func unixStamp(s string) *time.Time {
|
|
s = strings.TrimPrefix(strings.TrimSpace(s), "@")
|
|
n, err := strconv.ParseInt(s, 10, 64)
|
|
if err != nil || n == 0 {
|
|
return nil
|
|
}
|
|
t := time.Unix(n, 0)
|
|
return &t
|
|
}
|
|
|
|
// ScheduleNow is the schedule with what the service manager says of it; nil when the module has no
|
|
// schedule on this machine (no config file, or no cadence, day and time set).
|
|
func (m *Machine) ScheduleNow(ctx context.Context) (*Schedule, error) {
|
|
raw, err := m.ReadFile(ConfigFile)
|
|
if err != nil {
|
|
return nil, nil
|
|
}
|
|
c, refused := ParseConfig(raw, m.memMiB())
|
|
if len(refused) > 0 {
|
|
return nil, fmt.Errorf("%s refused %s %s", ConfigFile, refused[0].Key, refused[0].Reason)
|
|
}
|
|
s := ScheduleOf(c)
|
|
if s == nil {
|
|
return nil, nil
|
|
}
|
|
if out, err := m.ask(ctx, "systemctl", TimerShowArgv(s.Timer)...); err != nil {
|
|
s.Note = "the service manager did not say what the timer is: " + err.Error()
|
|
} else {
|
|
v := parseShow(out)
|
|
s.TimerState, s.Enabled = v["ActiveState"], v["UnitFileState"]
|
|
s.TimerNext, s.LastTrigger = unixStamp(v["NextElapseUSecRealtime"]), unixStamp(v["LastTriggerUSec"])
|
|
}
|
|
next := s.NextSlot(m.Now())
|
|
s.Next = &next
|
|
if out, err := m.ask(ctx, "systemctl", ServiceShowArgv(s.Service)...); err == nil {
|
|
v := parseShow(out)
|
|
s.ServiceState, s.LastResult, s.LastExit = v["ActiveState"], v["Result"], unixStamp(v["ExecMainExitTimestamp"])
|
|
}
|
|
return s, nil
|
|
}
|
|
|
|
// scheduleFindings are the schedule's warnings: a timer not waiting, a last run that failed with no scrub
|
|
// finished since, a slot past by MissedAfter that no scrub served.
|
|
func scheduleFindings(s *Schedule, p *PoolStatus, now time.Time, served map[string]int64) []string {
|
|
var warn []string
|
|
var scrubEnded *time.Time
|
|
if p != nil {
|
|
scrubEnded = LastScrubEnded(*p)
|
|
}
|
|
if slot := s.Slot(now); now.Sub(slot) > MissedAfter && served[s.Pool] < slot.Unix() && !startedSince(p, slot) {
|
|
warn = append(warn, fmt.Sprintf("The scheduled scrub of %s due %s has not started.", s.Pool, slot.Format("2006-01-02 15:04")))
|
|
}
|
|
if s.TimerState != "" && (s.TimerState != "active" || s.Enabled != "enabled") {
|
|
warn = append(warn, fmt.Sprintf("The scrub schedule's timer %s is %s and %s, so no scheduled scrub of %s will start.",
|
|
s.Timer, s.TimerState, s.Enabled, s.Pool))
|
|
}
|
|
if s.LastResult != "" && s.LastResult != "success" && s.ServiceState != "activating" &&
|
|
(scrubEnded == nil || s.LastExit == nil || scrubEnded.Before(*s.LastExit)) {
|
|
when := "at a time the service manager did not say"
|
|
if s.LastExit != nil {
|
|
when = s.LastExit.Local().Format("2006-01-02 15:04")
|
|
}
|
|
warn = append(warn, fmt.Sprintf("The last scheduled scrub of %s failed (%s), %s; its journal says why.", s.Pool, s.LastResult, when))
|
|
}
|
|
return warn
|
|
}
|
|
|
|
// ScrubEvent is the event each scrub act is published as (module.json's emits).
|
|
const ScrubEvent = "scrub.requested"
|
|
|
|
// ScrubArgv is each act's command: fixed words, the pool after `--`.
|
|
func ScrubArgv(act, pool string) []string {
|
|
switch act {
|
|
case "pause":
|
|
return []string{"scrub", "-p", "--", pool}
|
|
case "stop":
|
|
return []string{"scrub", "-s", "--", pool}
|
|
}
|
|
return []string{"scrub", "--", pool}
|
|
}
|
|
|
|
// ScrubActAnswer is what zfs_scrub_start, zfs_scrub_pause and zfs_scrub_stop answer.
|
|
type ScrubActAnswer struct {
|
|
Done bool `json:"done"`
|
|
Act string `json:"act"`
|
|
Pool string `json:"pool"`
|
|
Why string `json:"why"`
|
|
Ran []string `json:"ran"`
|
|
After string `json:"after"`
|
|
Recorded bool `json:"recorded"`
|
|
Record string `json:"record"`
|
|
Said string `json:"said"`
|
|
}
|
|
|
|
// ScrubAct starts, pauses or stops a pool's scrub, and answers what status says afterwards. A scrub reads
|
|
// every block and rewrites from redundancy only a block it finds damaged; it adds I/O load, and never
|
|
// changes what a file holds.
|
|
func (m *Machine) ScrubAct(ctx context.Context, act, pool, why string) (ScrubActAnswer, error) {
|
|
a := ScrubActAnswer{Act: act, Pool: pool, Why: why}
|
|
switch act {
|
|
case "start", "pause", "stop":
|
|
default:
|
|
return a, fmt.Errorf("%q is not an act of the scrub: start, pause or stop", act)
|
|
}
|
|
switch {
|
|
case why == "":
|
|
return a, fmt.Errorf("say why: a scrub act is a person's, and the why is kept with it")
|
|
case strings.ContainsAny(why, "\n\r") || len(why) > MaxWhy:
|
|
return a, fmt.Errorf("say why in one line of at most %d characters", MaxWhy)
|
|
}
|
|
ps, err := m.Pools(ctx)
|
|
if err != nil {
|
|
return a, err
|
|
}
|
|
known := false
|
|
for _, p := range ps {
|
|
known = known || p.Name == pool
|
|
}
|
|
if !known {
|
|
return a, fmt.Errorf("there is no pool %q on this machine; zfs_pools lists them", pool)
|
|
}
|
|
program, argv := m.Escalated("zpool", ScrubArgv(act, pool)...)
|
|
a.Ran = append([]string{program}, argv...)
|
|
_, runErr := m.AsRoot(ctx, "zpool", ScrubArgv(act, pool)...)
|
|
_, timedOut := runErr.(errTimedOut)
|
|
if st, err := m.Status(ctx); err == nil {
|
|
if p := statusOf(st, pool); p != nil {
|
|
a.After = p.ScansSaid()
|
|
}
|
|
}
|
|
a.Done = runErr == nil
|
|
switch {
|
|
case timedOut:
|
|
a.Said = fmt.Sprintf("zpool scrub did not answer within its time and was ended: outcome unknown; check zfs_scrub. Status now: %s.", a.After)
|
|
case runErr != nil:
|
|
a.Said = fmt.Sprintf("NOT DONE: %v. Status now: %s.", runErr, a.After)
|
|
default:
|
|
a.Said = fmt.Sprintf("Asked %s to %s its scrub. Status now: %s. Why: %s.", pool, act, a.After, strings.TrimSuffix(why, "."))
|
|
}
|
|
if m.Emit == nil {
|
|
a.Record = "this process publishes no events"
|
|
} else if err := m.Emit(ScrubEvent, map[string]any{"pool": pool, "act": act, "why": why, "done": a.Done,
|
|
"after": a.After, "at": m.Now().UTC().Format(time.RFC3339)}); err != nil {
|
|
a.Record = err.Error()
|
|
} else {
|
|
a.Recorded, a.Record = true, "published as the event "+ScrubEvent
|
|
}
|
|
if !a.Recorded {
|
|
a.Said += " NOT RECORDED on the bus: " + a.Record + "; the pool's own history (zpool history) has the command."
|
|
}
|
|
if runErr != nil {
|
|
return a, fmt.Errorf("%s", a.Said)
|
|
}
|
|
return a, nil
|
|
}
|