mesh-catalog/modules/zfs/cmd/zfs-tools/schedule.go

package main

import (
	"context"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"syscall"
	"time"
)

// The scrub schedule the module owns.
//
// The mesh writes ScrubConfig from the assignment's settings. The node-engine runs the module's
// scheduled process, `zfs-tools scrub-run`, from its own timer (ScrubTimer) every quarter of an hour, and
// the run decides from the configuration whether a scrub is due: the last scheduled time not after now
// (the slot) not yet served. Due, it starts the pool's scrub, or waits for one already running, as
// upstream's zfs-scrub@.service does; it never pauses or stops one, and nothing stops the unit to pause
// it (the node-engine writes no ExecStop). So no setting value ever reaches a unit file: the day and the
// time are read and checked here, and an invalid one is refused before anything runs.
//
// The mesh holds the distribution's own scrub timers for the pool stopped and disabled, so one schedule
// scrubs it; the node-engine gives them back running and enabled when the module goes (novox/hq ADR 0269).

// ScrubTimer and ScrubService are the node-engine's units for the scheduled run (the process's name).
const (
	ScrubTimer   = "zfs-mesh-scrub.timer"
	ScrubService = "zfs-mesh-scrub.service"
)

// ServedFile is where the scheduled run keeps the last slot it served, per pool: in a folder of root's
// alone (0700), apart from the operator account's state, so nothing the account runs can steer the run's
// writes (third review of mesh-catalog #147). The tools, which run as the account, cannot read it, and
// judge a missed scrub from status instead.
const ServedFile = "/var/lib/zfs-scrub/scrub-served.json"

// MissedAfter is how long after its slot a scrub that has not started is said to be missed: the run
// fires every quarter of an hour.
const MissedAfter = time.Hour

// ScrubConfig is the schedule as the mesh wrote it from the settings.
const ScrubConfig = "/etc/zfs-tools/scrub.conf"

// Margins after a cadence before a last scrub is said to be late: a weekly run that waited for a busy
// pool, a monthly run on the first given weekday of a month (28 or 35 days apart).
var cadenceAllows = map[string]time.Duration{
	"weekly":  9 * 24 * time.Hour,
	"monthly": 38 * 24 * time.Hour,
}

// Schedule is the pool's scrub schedule: what the settings say and what the service manager says of it.
type Schedule struct {
	Pool    string `json:"pool"`
	Cadence string `json:"cadence"`
	Day     string `json:"day"`
	Time    string `json:"time"`
	Timer   string `json:"timer"`
	Service string `json:"service"`
	// TimerState and Enabled are the timer's ActiveState and UnitFileState.
	TimerState string `json:"timer_state,omitempty"`
	Enabled    string `json:"enabled,omitempty"`
	// Next is the next scheduled scrub, from the settings; TimerNext when the timer next fires the run that
	// decides (every quarter of an hour).
	Next        *time.Time `json:"next_run,omitempty"`
	TimerNext   *time.Time `json:"timer_next,omitempty"`
	LastTrigger *time.Time `json:"last_triggered,omitempty"`
	// LastResult is the service's Result of its last run (success, exit-code, …), LastExit when it ended.
	LastResult   string     `json:"last_result,omitempty"`
	ServiceState string     `json:"service_state,omitempty"`
	LastExit     *time.Time `json:"last_run_ended,omitempty"`
	Note         string     `json:"note,omitempty"`
}

var (
	poolName = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_.:-]*$`)
	weekday  = regexp.MustCompile(`^(Mon|Tue|Wed|Thu|Fri|Sat|Sun)$`)
	clock    = regexp.MustCompile(`^([01][0-9]|2[0-3]):[0-5][0-9]$`)
)

// ParseScrubConfig reads the schedule: pool, cadence (weekly or monthly), day (Mon … Sun) and time (HH:MM).
func ParseScrubConfig(text string) (*Schedule, error) {
	v := map[string]string{}
	for _, l := range strings.Split(text, "\n") {
		l = strings.TrimSpace(l)
		if l == "" || strings.HasPrefix(l, "#") {
			continue
		}
		k, val, ok := strings.Cut(l, "=")
		if !ok {
			return nil, fmt.Errorf("%s: %q is not key=value", ScrubConfig, l)
		}
		v[strings.TrimSpace(k)] = strings.TrimSpace(val)
	}
	s := &Schedule{Pool: v["pool"], Cadence: v["cadence"], Day: v["day"], Time: v["time"]}
	switch {
	case !poolName.MatchString(s.Pool):
		return nil, fmt.Errorf("the scrub-pool setting %q is not a pool's name", s.Pool)
	case cadenceAllows[s.Cadence] == 0:
		return nil, fmt.Errorf("the scrub-cadence setting is %q; it is weekly or monthly", s.Cadence)
	case !weekday.MatchString(s.Day):
		return nil, fmt.Errorf("the scrub-day setting is %q; it is Mon, Tue, Wed, Thu, Fri, Sat or Sun", s.Day)
	case !clock.MatchString(s.Time):
		return nil, fmt.Errorf("the scrub-time setting is %q; it is HH:MM, 00:00 to 23:59", s.Time)
	}
	s.Timer, s.Service = ScrubTimer, ScrubService
	return s, nil
}

var weekdays = map[string]time.Weekday{"Sun": time.Sunday, "Mon": time.Monday, "Tue": time.Tuesday,
	"Wed": time.Wednesday, "Thu": time.Thursday, "Fri": time.Friday, "Sat": time.Saturday}

// at is the scheduled time on one day.
func (s *Schedule) at(day time.Time) time.Time {
	h, _ := strconv.Atoi(s.Time[:2])
	mi, _ := strconv.Atoi(s.Time[3:])
	return time.Date(day.Year(), day.Month(), day.Day(), h, mi, 0, 0, day.Location())
}

// scheduledOn says whether a scrub is scheduled on a day: its weekday, and for a monthly one the first
// such day of the month.
func (s *Schedule) scheduledOn(day time.Time) bool {
	return day.Weekday() == weekdays[s.Day] && (s.Cadence == "weekly" || day.Day() <= 7)
}

// Slot is the last scheduled time not after now.
func (s *Schedule) Slot(now time.Time) time.Time {
	day := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location())
	for i := 0; i < 45; i++ {
		if s.scheduledOn(day) && !s.at(day).After(now) {
			return s.at(day)
		}
		day = day.AddDate(0, 0, -1)
	}
	return time.Time{}
}

// NextSlot is the first scheduled time after now.
func (s *Schedule) NextSlot(now time.Time) time.Time {
	day := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location())
	for i := 0; i < 45; i++ {
		if s.scheduledOn(day) && s.at(day).After(now) {
			return s.at(day)
		}
		day = day.AddDate(0, 0, 1)
	}
	return time.Time{}
}

// WaitArgv waits for a pool's running scrub to end.
func WaitArgv(pool string) []string { return []string{"wait", "-t", "scrub", "--", pool} }

// WaitBound is how long the scheduled run waits for a scrub: past it the run ends and leaves the scrub
// running; a scrub never pauses for it.
const WaitBound = 7 * 24 * time.Hour

// readServed is the last slot served per pool; none when the file is not there or not readable.
func (m *Machine) readServed(path string) map[string]int64 {
	out := map[string]int64{}
	b, err := m.ReadFile(path)
	if err != nil {
		return out
	}
	_ = json.Unmarshal(b, &out)
	return out
}

// writeServed keeps the served slots, never through a link: the folder must be a real folder of this
// process's account alone (0700, made so when missing), and the file is written new beside it with
// O_NOFOLLOW and O_EXCL, then renamed over the old one, which replaces a link rather than following it.
func writeServed(path string, served map[string]int64) error {
	dir := filepath.Dir(path)
	if _, err := os.Lstat(dir); os.IsNotExist(err) {
		if err := os.Mkdir(dir, 0o700); err != nil {
			return err
		}
	}
	fi, err := os.Lstat(dir)
	if err != nil {
		return err
	}
	st, _ := fi.Sys().(*syscall.Stat_t)
	switch {
	case fi.Mode()&os.ModeSymlink != 0 || !fi.IsDir():
		return fmt.Errorf("%s is not a folder of its own (a link or a file): the served slot is not written", dir)
	case fi.Mode().Perm() != 0o700:
		return fmt.Errorf("%s is %v, not 0700: the served slot is not written", dir, fi.Mode().Perm())
	case st != nil && int(st.Uid) != os.Geteuid():
		return fmt.Errorf("%s belongs to uid %d, not this process's %d: the served slot is not written", dir, st.Uid, os.Geteuid())
	}
	if fi, err := os.Lstat(path); err == nil && !fi.Mode().IsRegular() {
		return fmt.Errorf("%s is not a plain file (a link or something else): the served slot is not written", path)
	}
	b, _ := json.Marshal(served)
	tmp := path + ".new"
	_ = os.Remove(tmp)
	f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_EXCL|syscall.O_NOFOLLOW, 0o600)
	if err != nil {
		return err
	}
	if _, err := f.Write(b); err != nil {
		f.Close()
		return err
	}
	if err := f.Close(); err != nil {
		return err
	}
	return os.Rename(tmp, path)
}

// startedSince says whether status shows the pool's scrub started at or after a time.
func startedSince(p *PoolStatus, slot time.Time) bool {
	if p == nil {
		return false
	}
	sc := p.Scrub()
	return sc != nil && sc.Started != nil && !sc.Started.Before(slot)
}

// ScrubRun is the scheduled run: a due scrub started, or one running waited for. It answers what it did in
// a line, and fails on an invalid schedule, which it never acts on.
func (m *Machine) ScrubRun(ctx context.Context, servedPath string) (string, error) {
	raw, err := m.ReadFile(ScrubConfig)
	if err != nil {
		return "", fmt.Errorf("%s could not be read: %v", ScrubConfig, err)
	}
	s, err := ParseScrubConfig(string(raw))
	if err != nil {
		return "", fmt.Errorf("refused, nothing run: %v", err)
	}
	now := m.Now()
	slot := s.Slot(now)
	served := m.readServed(servedPath)
	if served[s.Pool] >= slot.Unix() {
		return fmt.Sprintf("the scrub of %s scheduled %s was served", s.Pool, slot.Format("2006-01-02 15:04")), nil
	}
	st, err := m.Status(ctx)
	if err != nil {
		return "", err
	}
	p := statusOf(st, s.Pool)
	if p == nil {
		return "", fmt.Errorf("the scheduled pool %s is not imported here", s.Pool)
	}
	mark := func() error {
		served[s.Pool] = slot.Unix()
		return writeServed(servedPath, served)
	}
	if startedSince(p, slot) && p.Scrub().State != "running" {
		return fmt.Sprintf("the scrub of %s scheduled %s ran already", s.Pool, slot.Format("2006-01-02 15:04")), mark()
	}
	for _, r := range p.Resilvers() {
		if r.State == "running" {
			return fmt.Sprintf("a resilver of %s runs; the scrub scheduled %s waits for the next run after it", s.Pool,
				slot.Format("2006-01-02 15:04")), nil
		}
	}
	started := false
	if sc := p.Scrub(); sc == nil || sc.State != "running" {
		_, err := m.AsRoot(ctx, "zpool", ScrubArgv("start", s.Pool)...)
		switch {
		case err == nil:
			started = true
		case strings.Contains(err.Error(), "currently scrubbing"):
			// One began since status was read: it serves the slot, and is waited for like any other.
		default:
			return "", fmt.Errorf("the scheduled scrub of %s could not start: %v", s.Pool, err)
		}
	}
	if err := mark(); err != nil {
		return "", fmt.Errorf("the scrub of %s runs, and the served slot could not be kept: %v", s.Pool, err)
	}
	wctx, cancel := context.WithTimeout(ctx, WaitBound)
	defer cancel()
	_, werr := m.AsRoot(wctx, "zpool", WaitArgv(s.Pool)...)
	what := "waited for the scrub of " + s.Pool + " already running"
	if started {
		what = "started the scrub of " + s.Pool + " scheduled " + slot.Format("2006-01-02 15:04") + ", and waited for it"
	}
	if werr != nil {
		return what + "; the wait ended without its end (" + werr.Error() + "), and the scrub goes on", nil
	}
	return what + " to finish", nil
}

// TimerShowArgv asks the service manager what a timer says: whether it runs and is enabled, when it
// fires next and when it last did, as Unix times.
func TimerShowArgv(unit string) []string {
	return []string{"show", "--timestamp=unix", "--property=ActiveState,UnitFileState,NextElapseUSecRealtime,LastTriggerUSec", "--", unit}
}

// ServiceShowArgv asks what the scrub service's last run was.
func ServiceShowArgv(unit string) []string {
	return []string{"show", "--timestamp=unix", "--property=ActiveState,Result,ExecMainStartTimestamp,ExecMainExitTimestamp", "--", unit}
}

// parseShow reads `systemctl show`'s key=value lines.
func parseShow(out string) map[string]string {
	v := map[string]string{}
	for _, l := range lines(out) {
		if k, val, ok := strings.Cut(l, "="); ok {
			v[k] = strings.TrimSpace(val)
		}
	}
	return v
}

// unixStamp reads `--timestamp=unix`: "@1791842400"; nil for "", "n/a" or 0.
func unixStamp(s string) *time.Time {
	s = strings.TrimPrefix(strings.TrimSpace(s), "@")
	n, err := strconv.ParseInt(s, 10, 64)
	if err != nil || n == 0 {
		return nil
	}
	t := time.Unix(n, 0)
	return &t
}

// ScheduleNow is the schedule with what the service manager says of it; nil when the module has no
// schedule on this machine (no configuration written).
func (m *Machine) ScheduleNow(ctx context.Context) (*Schedule, error) {
	raw, err := m.ReadFile(ScrubConfig)
	if err != nil {
		return nil, nil
	}
	s, err := ParseScrubConfig(string(raw))
	if err != nil {
		return nil, err
	}
	if out, err := m.ask(ctx, "systemctl", TimerShowArgv(s.Timer)...); err != nil {
		s.Note = "the service manager did not say what the timer is: " + err.Error()
	} else {
		v := parseShow(out)
		s.TimerState, s.Enabled = v["ActiveState"], v["UnitFileState"]
		s.TimerNext, s.LastTrigger = unixStamp(v["NextElapseUSecRealtime"]), unixStamp(v["LastTriggerUSec"])
	}
	next := s.NextSlot(m.Now())
	s.Next = &next
	if out, err := m.ask(ctx, "systemctl", ServiceShowArgv(s.Service)...); err == nil {
		v := parseShow(out)
		s.ServiceState, s.LastResult, s.LastExit = v["ActiveState"], v["Result"], unixStamp(v["ExecMainExitTimestamp"])
	}
	return s, nil
}

// scheduleFindings are the schedule's warnings: a timer not waiting, a last run that failed with no scrub
// finished since, a slot past by MissedAfter that no scrub served.
func scheduleFindings(s *Schedule, p *PoolStatus, now time.Time, served map[string]int64) []string {
	var warn []string
	var scrubEnded *time.Time
	if p != nil {
		scrubEnded = LastScrubEnded(*p)
	}
	if slot := s.Slot(now); now.Sub(slot) > MissedAfter && served[s.Pool] < slot.Unix() && !startedSince(p, slot) {
		warn = append(warn, fmt.Sprintf("The scheduled scrub of %s due %s has not started.", s.Pool, slot.Format("2006-01-02 15:04")))
	}
	if s.TimerState != "" && (s.TimerState != "active" || s.Enabled != "enabled") {
		warn = append(warn, fmt.Sprintf("The scrub schedule's timer %s is %s and %s, so no scheduled scrub of %s will start.",
			s.Timer, s.TimerState, s.Enabled, s.Pool))
	}
	if s.LastResult != "" && s.LastResult != "success" && s.ServiceState != "activating" &&
		(scrubEnded == nil || s.LastExit == nil || scrubEnded.Before(*s.LastExit)) {
		when := "at a time the service manager did not say"
		if s.LastExit != nil {
			when = s.LastExit.Local().Format("2006-01-02 15:04")
		}
		warn = append(warn, fmt.Sprintf("The last scheduled scrub of %s failed (%s), %s; its journal says why.", s.Pool, s.LastResult, when))
	}
	return warn
}

// ScrubEvent is the event each scrub act is published as (module.json's emits).
const ScrubEvent = "scrub.requested"

// ScrubArgv is each act's command: fixed words, the pool after `--`.
func ScrubArgv(act, pool string) []string {
	switch act {
	case "pause":
		return []string{"scrub", "-p", "--", pool}
	case "stop":
		return []string{"scrub", "-s", "--", pool}
	}
	return []string{"scrub", "--", pool}
}

// ScrubActAnswer is what zfs_scrub_start, zfs_scrub_pause and zfs_scrub_stop answer.
type ScrubActAnswer struct {
	Done     bool     `json:"done"`
	Act      string   `json:"act"`
	Pool     string   `json:"pool"`
	Why      string   `json:"why"`
	Ran      []string `json:"ran"`
	After    string   `json:"after"`
	Recorded bool     `json:"recorded"`
	Record   string   `json:"record"`
	Said     string   `json:"said"`
}

// ScrubAct starts, pauses or stops a pool's scrub, and answers what status says afterwards. A scrub reads
// every block and rewrites from redundancy only a block it finds damaged; it adds I/O load, and never
// changes what a file holds.
func (m *Machine) ScrubAct(ctx context.Context, act, pool, why string) (ScrubActAnswer, error) {
	a := ScrubActAnswer{Act: act, Pool: pool, Why: why}
	switch act {
	case "start", "pause", "stop":
	default:
		return a, fmt.Errorf("%q is not an act of the scrub: start, pause or stop", act)
	}
	switch {
	case why == "":
		return a, fmt.Errorf("say why: a scrub act is a person's, and the why is kept with it")
	case strings.ContainsAny(why, "\n\r") || len(why) > MaxWhy:
		return a, fmt.Errorf("say why in one line of at most %d characters", MaxWhy)
	}
	ps, err := m.Pools(ctx)
	if err != nil {
		return a, err
	}
	known := false
	for _, p := range ps {
		known = known || p.Name == pool
	}
	if !known {
		return a, fmt.Errorf("there is no pool %q on this machine; zfs_pools lists them", pool)
	}
	program, argv := m.Escalated("zpool", ScrubArgv(act, pool)...)
	a.Ran = append([]string{program}, argv...)
	_, runErr := m.AsRoot(ctx, "zpool", ScrubArgv(act, pool)...)
	_, timedOut := runErr.(errTimedOut)
	if st, err := m.Status(ctx); err == nil {
		if p := statusOf(st, pool); p != nil {
			a.After = p.ScansSaid()
		}
	}
	a.Done = runErr == nil
	switch {
	case timedOut:
		a.Said = fmt.Sprintf("zpool scrub did not answer within its time and was ended: outcome unknown; check zfs_scrub. Status now: %s.", a.After)
	case runErr != nil:
		a.Said = fmt.Sprintf("NOT DONE: %v. Status now: %s.", runErr, a.After)
	default:
		a.Said = fmt.Sprintf("Asked %s to %s its scrub. Status now: %s. Why: %s.", pool, act, a.After, strings.TrimSuffix(why, "."))
	}
	if m.Emit == nil {
		a.Record = "this process publishes no events"
	} else if err := m.Emit(ScrubEvent, map[string]any{"pool": pool, "act": act, "why": why, "done": a.Done,
		"after": a.After, "at": m.Now().UTC().Format(time.RFC3339)}); err != nil {
		a.Record = err.Error()
	} else {
		a.Recorded, a.Record = true, "published as the event "+ScrubEvent
	}
	if !a.Recorded {
		a.Said += " NOT RECORDED on the bus: " + a.Record + "; the pool's own history (zpool history) has the command."
	}
	if runErr != nil {
		return a, fmt.Errorf("%s", a.Said)
	}
	return a, nil
}