Z10 zfs-run: the scheduled run takes the lock, starts a due scrub and trim without waiting, and acts on hand acts

Builds: design 42 zfs-owns-zfs §20, §24, A-17
This commit is contained in:
2026-11-13 00:38:17 +01:00
parent 1bea6edcb8
commit 06d83912cd
10 changed files with 502 additions and 462 deletions
+5
View File
@@ -149,6 +149,11 @@ func (m *Machine) Apply(ctx context.Context) (string, error) {
return "", fmt.Errorf("the apply's lock %s: %v", ApplyLock, err)
}
defer release()
return m.applyLocked(ctx)
}
// applyLocked is the apply under a lock already held (the scheduled run's hand act takes the same lock).
func (m *Machine) applyLocked(ctx context.Context) (string, error) {
text, err := m.ReadFile(ConfigFile)
if err != nil {
return "", fmt.Errorf("the config file %s cannot be read: %v", ConfigFile, err)
+11 -5
View File
@@ -6,8 +6,8 @@
// speaks MCP to over stdio (novox/hq ADR 0188, ADR 0193), as the operator account. With `watch` and
// `events` it is one of the module's two long-running processes, run by the node-engine: the watch, which
// the health check zfs_health stands on (watch.go), and the event follower, as root, judged by its unit
// (follow.go). With `scrub-run` it is the scheduled scrub, run as root by the node-engine's timer
// (schedule.go).
// (follow.go). With `run` it is the scheduled run, as root by the node-engine's timer (run.go), and with `apply` the
// apply that converges the node to its config file (apply.go).
package main
import (
@@ -27,8 +27,14 @@ func main() {
switch os.Args[1] {
case "watch":
(&Watch{M: Here(), Out: os.Stdout}).Loop(ctx)
case "scrub-run":
said, err := Here().ScrubRun(ctx, ServedFile)
case "run":
m := Here()
closeBus := m.JoinBus()
said, err := m.RunScheduled(ctx, ServedFile)
closeBus()
for _, w := range m.Warnings {
fmt.Fprintln(os.Stderr, "warning:", w)
}
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
@@ -54,7 +60,7 @@ func main() {
}
default:
fmt.Fprintf(os.Stderr, "zfs-tools: %q is not a way to run; with no argument it serves its tools, with watch it "+
"watches the pools, with events it follows ZFS's events, with scrub-run it runs the scheduled scrub, with apply it converges the node to its config file\n", os.Args[1])
"watches the pools, with events it follows ZFS's events, with run it runs the scheduled scrub, trim and snapshots, with apply it converges the node to its config file\n", os.Args[1])
os.Exit(2)
}
return
+14 -12
View File
@@ -119,10 +119,10 @@ func TestExactlyOneScrubScheduleAndNoSettingInAUnitFile(t *testing.T) {
byID[r["id"].(string)] = r
order[r["id"].(string)] = i
}
run := byID["scrub-run"]
run := byID["run"]
args, _ := run["run"].([]any)
if run["type"] != "process" || run["name"] != "zfs-mesh-scrub" || run["schedule"] != "0,15,30,45 * * * *" || run["user"] != nil ||
len(args) != 2 || args[1] != "scrub-run" {
if run["type"] != "process" || run["name"] != "zfs-mesh-run" || run["schedule"] != "0,15,30,45 * * * *" || run["user"] != nil ||
len(args) != 2 || args[1] != "run" {
t.Errorf("the scheduled run: %v", run)
}
// The node-engine writes a cron step ("*/15") as a calendar systemd refuses ("*:*/15"), and the timer never
@@ -131,15 +131,15 @@ func TestExactlyOneScrubScheduleAndNoSettingInAUnitFile(t *testing.T) {
if s, _ := run["schedule"].(string); strings.Contains(s, "/") {
t.Errorf("the scheduled run's schedule %q uses a step, which the node-engine writes as a calendar systemd refuses", s)
}
if ScrubTimer != run["name"].(string)+".timer" || ScrubService != run["name"].(string)+".service" {
t.Errorf("the tools read %s and %s; the node-engine names them after %v", ScrubTimer, ScrubService, run["name"])
if RunTimer != run["name"].(string)+".timer" || RunService != run["name"].(string)+".service" {
t.Errorf("the tools read %s and %s; the node-engine names them after %v", RunTimer, RunService, run["name"])
}
for _, c := range []string{"weekly", "monthly"} {
d := byID["distribution-"+c]
if d["unit"] != "zfs-scrub-"+c+"@${config:scrub-pool}.timer" || d["state"] != "stopped" || d["boot"] != "disabled" {
t.Errorf("the distribution's %s timer: %v", c, d)
}
if order["scrub-run"] > order["distribution-"+c] {
if order["run"] > order["distribution-"+c] {
t.Errorf("the distribution's %s timer is stopped before the run is installed", c)
}
}
@@ -188,7 +188,7 @@ func TestTheHomeServersComposedUnitsAndConfiguration(t *testing.T) {
conf = fill.Replace(templateText(t, r))
}
}
want := []string{"zfs-mesh-scrub.timer running/enabled (the node-engine's, every quarter of an hour)",
want := []string{"zfs-mesh-run.timer running/enabled (the node-engine's, every quarter of an hour)",
"zfs-scrub-weekly@storage.timer stopped/disabled", "zfs-scrub-monthly@storage.timer stopped/disabled"}
if strings.Join(units, "\n") != strings.Join(want, "\n") {
t.Errorf("units:\n%s", strings.Join(units, "\n"))
@@ -196,9 +196,11 @@ func TestTheHomeServersComposedUnitsAndConfiguration(t *testing.T) {
for _, u := range units {
t.Log(u)
}
s, err := ParseScrubConfig(conf)
if err != nil || s.Pool != "storage" || s.Cadence != "monthly" {
t.Fatalf("configuration %q: %v", conf, err)
_ = conf
cfg, refused := ParseConfig([]byte(scrubConf("storage", "monthly", "Mon", "00:00")), 16000)
s := ScheduleOf(cfg)
if len(refused) != 0 || s == nil || s.Pool != "storage" || s.Cadence != "monthly" {
t.Fatalf("configuration: %v", refused)
}
t.Logf("next scrub of storage after 2026-10-09 12:00: %s", s.NextSlot(time.Date(2026, 10, 9, 12, 0, 0, 0, time.Local)).Format("2006-01-02 15:04"))
if next := s.NextSlot(time.Date(2026, 10, 9, 12, 0, 0, 0, time.Local)); !next.Equal(time.Date(2026, 11, 2, 0, 0, 0, 0, time.Local)) {
@@ -216,7 +218,7 @@ func TestTheBundleRunsOnlyReadingVerbsSnapshotAndScrub(t *testing.T) {
"SnapshotArgv": {"zfs", "snapshot"},
"PoolsArgv": {"zpool", "list"}, "StatusArgv": {"zpool", "status"}, "IOArgv": {"zpool", "iostat"},
"EventsArgv": {"zpool", "events"}, "HistoryArgv": {"zpool", "history"}, "FollowArgv": {"zpool", "events"},
"ScrubArgv": {"zpool", "scrub"}, "WaitArgv": {"zpool", "wait"},
"ScrubArgv": {"zpool", "scrub"}, "TrimArgv": {"zpool", "trim"},
"TimerShowArgv": {"systemctl", "show"}, "ServiceShowArgv": {"systemctl", "show"},
"PoolGetArgv": {"zpool", "get"}, "PoolSetArgv": {"zpool", "set"}, "DatasetGetArgv": {"zfs", "get"},
"DatasetSetArgv": {"zfs", "set"}, "DatasetInheritArgv": {"zfs", "inherit"},
@@ -227,7 +229,7 @@ func TestTheBundleRunsOnlyReadingVerbsSnapshotAndScrub(t *testing.T) {
verbs := map[string][]string{"DatasetsArgv": DatasetsArgv(), "SnapshotsArgv": SnapshotsArgv(), "PoolsArgv": PoolsArgv(),
"StatusArgv": StatusArgv(), "IOArgv": IOArgv(3), "EventsArgv": EventsArgv(), "HistoryArgv": HistoryArgv("tank"),
"PropertiesArgv": PropertiesArgv("tank"), "SnapshotArgv": SnapshotArgv("tank", "x"), "FollowArgv": FollowArgv(),
"TimerShowArgv": TimerShowArgv("x.timer"), "ServiceShowArgv": ServiceShowArgv("x.service"), "WaitArgv": WaitArgv("tank"),
"TimerShowArgv": TimerShowArgv("x.timer"), "ServiceShowArgv": ServiceShowArgv("x.service"), "TrimArgv": TrimArgv("tank"),
"PoolGetArgv": PoolGetArgv("tank", "autotrim"), "PoolSetArgv": PoolSetArgv("tank", "autotrim", "on"),
"DatasetGetArgv": DatasetGetArgv("tank", "atime"), "DatasetSetArgv": DatasetSetArgv("tank", "atime", "on"),
"DatasetInheritArgv": DatasetInheritArgv("tank", "atime"),
+168
View File
@@ -0,0 +1,168 @@
package main
import (
"context"
"fmt"
"sort"
"strings"
)
// The scheduled run (design 42 zfs-owns-zfs §20, §24).
//
// `zfs-tools run` is the module's one scheduled run, root's, fired by the node-engine's timer every quarter of
// an hour. Each run takes the apply's lock; held, it does nothing and says `run.skipped` {reason: lock-held}.
// It does nothing while a scrub or resilver runs or before the pool is imported, saying that reason.
// Otherwise it acts on the hand acts waiting in the state `schedule`, starts the scrub when due and returns
// at once (the scrub's end is the event follower's to hear, §27), and starts a trim when trim-schedule is due.
// The reasons of `run.skipped`.
const (
SkipLockHeld = "lock-held"
SkipPoolNotImported = "pool-not-imported"
SkipScanRunning = "scrub-or-resilver-running"
)
// KeyPaused is the key of the state `schedule` that holds the schedule's snapshots (§24).
const KeyPaused = "paused"
// TrimArgv starts a trim of the pool.
func TrimArgv(pool string) []string { return []string{"trim", "--", pool} }
// handActs are the acts waiting in the state `schedule`, each under its own key (A-17), but the pause.
func (m *Machine) handActs() []string {
if m.States == nil {
return nil
}
keys, err := m.States.Keys(StateSchedule)
if err != nil {
return nil
}
var acts []string
for _, k := range keys {
if k != KeyPaused {
acts = append(acts, k)
}
}
sort.Strings(acts)
return acts
}
func (m *Machine) skip(reason string) string {
var failed []error
if err := m.say(EventRunSkipped, map[string]any{"reason": reason}); err != nil {
m.failure(&failed, "run.skipped", err)
}
return "skipped: " + reason
}
// RunScheduled is the scheduled run. It answers what it did in a line; its error is a command that failed.
func (m *Machine) RunScheduled(ctx context.Context, servedPath string) (string, error) {
release, got, err := m.Lock(ApplyLock, false)
if err != nil {
return "", fmt.Errorf("the lock %s: %v", ApplyLock, err)
}
if !got {
return m.skip(SkipLockHeld), nil
}
defer release()
text, err := m.ReadFile(ConfigFile)
if err != nil {
return "", fmt.Errorf("%s could not be read: %v", ConfigFile, err)
}
c, refused := ParseConfig(text, m.memMiB())
if len(refused) > 0 {
return "refused, nothing run: " + refused[0].Key + " " + refused[0].Reason, nil
}
var said []string
var failed []error
acts := m.handActs()
guard := ""
var p *PoolStatus
if c.Pool != "" {
wait, perr := m.poolWait(ctx, c.Pool)
switch {
case perr != nil:
return "", perr
case strings.Contains(wait, "to be imported"):
guard = SkipPoolNotImported
case wait != "":
guard = SkipScanRunning
}
if guard == "" {
st, err := m.Status(ctx)
if err != nil {
return "", err
}
p = statusOf(st, c.Pool)
}
}
for _, act := range acts {
switch {
case act == "apply":
s, err := m.applyLocked(ctx)
said = append(said, "apply: "+s)
if err != nil {
failed = append(failed, err)
}
case guard != "":
said = append(said, m.skip(guard))
default:
m.Warnings = append(m.Warnings, "the hand act "+act+" is not one this run knows")
}
_ = m.States.Delete(StateSchedule, act)
}
if guard != "" || c.Pool == "" {
if guard != "" && len(acts) == 0 {
said = append(said, m.skip(guard))
}
return strings.Join(said, "; "), joinErrors(failed)
}
served := m.readServed(servedPath)
now := m.Now()
for _, work := range []struct {
name, key, verb string
s *Schedule
start func() error
}{{"scrub", c.Pool, "start", ScheduleOf(c), func() error {
_, err := m.AsRoot(ctx, "zpool", ScrubArgv("start", c.Pool)...)
return err
}}, {"trim", c.Pool + "/trim", "start", TrimScheduleOf(c), func() error {
_, err := m.AsRoot(ctx, "zpool", TrimArgv(c.Pool)...)
return err
}}} {
if work.s == nil {
continue
}
slot := work.s.Slot(now)
if slot.IsZero() || served[work.key] >= slot.Unix() {
continue
}
if work.name == "scrub" && startedSince(p, slot) {
said = append(said, fmt.Sprintf("the scrub of %s scheduled %s ran already", c.Pool, slot.Format("2006-01-02 15:04")))
} else if err := work.start(); err != nil && !strings.Contains(err.Error(), "currently") {
failed = append(failed, fmt.Errorf("the scheduled %s of %s could not start: %v", work.name, c.Pool, err))
continue
} else {
said = append(said, fmt.Sprintf("started the %s of %s scheduled %s", work.name, c.Pool, slot.Format("2006-01-02 15:04")))
}
served[work.key] = slot.Unix()
if err := writeServed(servedPath, served); err != nil {
failed = append(failed, fmt.Errorf("the %s of %s was started, and the served slot could not be kept: %v", work.name, c.Pool, err))
}
}
if len(said) == 0 {
said = append(said, "nothing due")
}
return strings.Join(said, "; "), joinErrors(failed)
}
func joinErrors(errs []error) error {
if len(errs) == 0 {
return nil
}
var parts []string
for _, e := range errs {
parts = append(parts, e.Error())
}
return fmt.Errorf("%s", strings.Join(parts, "; "))
}
+269
View File
@@ -0,0 +1,269 @@
package main
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
func scrubConf(pool, cadence, day, at string) string {
return `{"pool":"` + pool + `","scrub-cadence":"` + cadence + `","scrub-day":"` + day + `","scrub-time":"` + at + `"}`
}
func TestTheSlotIsTheLastScheduledTimeNotAfterNow(t *testing.T) {
loc := time.Local
at := func(y int, m time.Month, d, h, min int) time.Time { return time.Date(y, m, d, h, min, 0, 0, loc) }
cases := []struct {
conf string
now time.Time
want time.Time
}{
{scrubConf("tank", "weekly", "Mon", "00:00"), at(2026, 10, 9, 12, 0), at(2026, 10, 5, 0, 0)},
{scrubConf("tank", "weekly", "Fri", "13:30"), at(2026, 10, 9, 12, 0), at(2026, 10, 2, 13, 30)},
{scrubConf("tank", "weekly", "Fri", "11:30"), at(2026, 10, 9, 12, 0), at(2026, 10, 9, 11, 30)},
{scrubConf("tank", "monthly", "Mon", "00:00"), at(2026, 10, 9, 12, 0), at(2026, 10, 5, 0, 0)},
{scrubConf("tank", "monthly", "Mon", "00:00"), at(2026, 10, 4, 23, 0), at(2026, 9, 7, 0, 0)},
{scrubConf("tank", "monthly", "Thu", "06:00"), at(2026, 10, 1, 6, 0), at(2026, 10, 1, 6, 0)},
}
for _, c := range cases {
cfg, ref := ParseConfig([]byte(c.conf), 16000)
if len(ref) != 0 {
t.Fatal(ref)
}
s := ScheduleOf(cfg)
if got := s.Slot(c.now); !got.Equal(c.want) {
t.Errorf("%q at %s: slot %s, want %s", c.conf, c.now, got, c.want)
}
if next := s.NextSlot(c.now); !next.After(c.now) || !s.Slot(next).Equal(next) {
t.Errorf("%q at %s: next slot %s", c.conf, c.now, next)
}
}
// A schedule with a value unset is no schedule; a value of another form is refused.
if cfg, _ := ParseConfig([]byte(`{"pool":"tank","scrub-cadence":"weekly"}`), 16000); ScheduleOf(cfg) != nil {
t.Error("a schedule without day and time")
}
for _, conf := range []string{scrubConf("tank", "daily", "Mon", "00:00"), scrubConf("tank", "weekly", "Monday", "00:00"),
scrubConf("tank", "weekly", "Mon", "25:00"), scrubConf("tank x", "weekly", "Mon", "00:00"), scrubConf("tank", "weekly", "Mon..Fri", "00:00"),
scrubConf("tank", "weekly", "Mon", "*:*")} {
if _, ref := ParseConfig([]byte(conf), 16000); len(ref) == 0 {
t.Errorf("accepted %q", conf)
}
}
}
// runFake is a root machine on the healthy fixtures, with the config file, the schedule state and the lock.
func runFake(t *testing.T, conf string) (*fake, *Machine, memStates, string) {
f := withZFS(t)
f.uid = 0
f.files[ConfigFile] = conf
f.files[MemInfoFile] = "MemTotal: 16384000 kB\n"
m := f.machine()
st := memStates{}
m.States = st
return f, m, st, filepath.Join(t.TempDir(), "zfs-scrub", "scrub-served.json")
}
// §20, §24, §31: the lock held: nothing run; a scrub or resilver running or an unimported pool: nothing run
// and said; a due scrub started and the run ends without waiting; a due trim by trim-day and trim-time; a
// served slot not run again; a hand act's key acted on under the lock and deleted.
func TestTheScheduledRunDoesWhatIsDueAndEndsWithoutWaiting(t *testing.T) {
const trim = `{"pool":"tank","trim-schedule":"weekly","trim-day":"Mon","trim-time":"00:00"}`
cases := []struct {
name, conf string
setup func(f *fake, st memStates)
ran, not []string
event string // event kind and reason
served bool
said string
wantErr bool
}{
{name: "scrub due", conf: scrubConf("tank", "monthly", "Mon", "00:00"), setup: func(f *fake, _ memStates) { f.on("zpool scrub -- tank", "") },
ran: []string{"zpool scrub -- tank"}, not: []string{"zpool wait"}, served: true, said: "started the scrub"},
{name: "scrub refused as one just began", conf: scrubConf("tank", "monthly", "Mon", "00:00"), setup: func(f *fake, _ memStates) {
f.answers["zpool scrub -- tank"] = Ran{Code: 1, Err: "cannot scrub tank: currently scrubbing"}
}, ran: []string{"zpool scrub -- tank"}, not: []string{"zpool wait", "scrub -p", "scrub -s"}, served: true},
{name: "scrub running", conf: scrubConf("tank", "monthly", "Mon", "00:00"), setup: func(f *fake, _ memStates) {
f.on(line("zpool", StatusArgv()), fixture(t, "zpool-status-scrubbing.txt"))
}, not: []string{"zpool scrub", "zpool wait"}, event: "run.skipped/" + SkipScanRunning},
{name: "resilver running", conf: scrubConf("tank", "monthly", "Mon", "00:00"), setup: func(f *fake, _ memStates) {
f.on(line("zpool", StatusArgv()), fixture(t, "zpool-status-degraded.txt"))
}, not: []string{"zpool scrub"}, event: "run.skipped/" + SkipScanRunning},
{name: "pool not imported", conf: scrubConf("nopool", "monthly", "Mon", "00:00"), not: []string{"zpool scrub"},
event: "run.skipped/" + SkipPoolNotImported},
{name: "slot already served by a scrub that started in it", conf: scrubConf("tank", "weekly", "Sun", "00:00"), setup: func(f *fake, _ memStates) {
f.now = time.Date(2026, 9, 15, 12, 0, 0, 0, time.Local)
}, not: []string{"zpool scrub"}, served: true, said: "ran already"},
{name: "trim due", conf: trim, setup: func(f *fake, _ memStates) { f.on("zpool trim -- tank", "") },
ran: []string{"zpool trim -- tank"}, served: true, said: "started the trim"},
{name: "trim none", conf: `{"pool":"tank","trim-schedule":"none","trim-day":"Mon","trim-time":"00:00"}`, not: []string{"zpool trim"}},
{name: "a hand act's key is acted on and deleted", conf: `{"pool":"tank"}`, setup: func(f *fake, st memStates) {
st.Put(StateSchedule, "apply", map[string]string{"by": "x"})
}, said: "apply: applied"},
{name: "a refused config runs nothing", conf: `{"pool":"tank","scrub-cadence":"hourly"}`, not: []string{"zpool"}, said: "refused"},
{name: "a failed start fails the run", conf: scrubConf("tank", "monthly", "Mon", "00:00"), setup: func(f *fake, _ memStates) {
f.answers["zpool scrub -- tank"] = Ran{Code: 1, Err: "cannot scrub: no such pool"}
}, ran: []string{"zpool scrub -- tank"}, wantErr: true},
}
for _, c := range cases {
f, m, st, served := runFake(t, c.conf)
if c.setup != nil {
c.setup(f, st)
}
said, err := m.RunScheduled(context.Background(), served)
if (err != nil) != c.wantErr {
t.Errorf("%s: error %v", c.name, err)
}
for _, r := range c.ran {
if len(ran(f, r)) != 1 {
t.Errorf("%s: %q not run once; ran %v", c.name, r, f.calls)
}
}
for _, n := range c.not {
if len(ran(f, n)) != 0 {
t.Errorf("%s: ran %v", c.name, ran(f, n))
}
}
if c.event != "" {
kind, reason, _ := strings.Cut(c.event, "/")
if len(f.emitted) != 1 || f.emitted[0]["_type"] != kind || f.emitted[0]["reason"] != reason {
t.Errorf("%s: events %v, want %s", c.name, f.emitted, c.event)
}
}
if _, err := os.Stat(served); (err == nil) != c.served {
t.Errorf("%s: served file present %v, want %v", c.name, err == nil, c.served)
}
if !strings.Contains(said, c.said) {
t.Errorf("%s: said %q, want %q", c.name, said, c.said)
}
if _, ok := st["schedule/apply"]; ok {
t.Errorf("%s: the hand act's key stayed", c.name)
}
}
// The lock held by another: nothing run, said.
f, m, _, served := runFake(t, scrubConf("tank", "monthly", "Mon", "00:00"))
m.Lock = func(string, bool) (func(), bool, error) { return nil, false, nil }
if said, err := m.RunScheduled(context.Background(), served); err != nil || !strings.Contains(said, SkipLockHeld) ||
len(f.emitted) != 1 || f.emitted[0]["reason"] != SkipLockHeld || len(ran(f, "zpool")) != 0 {
t.Errorf("lock held: %q %v %v %v", said, err, f.emitted, f.calls)
}
}
// scheduled is the healthy fixtures with the module's scrub schedule for tank, its timer waiting and its
// last run a success.
func scheduled(t *testing.T) *fake {
f := withZFS(t)
f.files[ConfigFile] = scrubConf("tank", "weekly", "Mon", "00:00")
f.files[MemInfoFile] = "MemTotal: 16384000 kB\n"
f.on(line("systemctl", TimerShowArgv(RunTimer)),
"ActiveState=active\nUnitFileState=enabled\nNextElapseUSecRealtime=@1791842400\nLastTriggerUSec=@1791237600\n")
f.on(line("systemctl", ServiceShowArgv(RunService)),
"ActiveState=inactive\nResult=success\nExecMainStartTimestamp=@1791237600\nExecMainExitTimestamp=@1791331200\n")
return f
}
func TestTheScheduleIsTheConfigAndWhatTheTimerSays(t *testing.T) {
f := scheduled(t)
got, err := f.machine().Scrubs(context.Background(), false)
if err != nil {
t.Fatal(err)
}
a := got.(ScrubAnswer)
s := a.Schedule
if s == nil || s.Pool != "tank" || s.Cadence != "weekly" || s.Timer != "zfs-mesh-run.timer" || s.TimerState != "active" ||
s.Enabled != "enabled" || s.Next == nil || !s.Next.Equal(time.Date(2026, 10, 12, 0, 0, 0, 0, time.Local)) || s.LastResult != "success" {
t.Fatalf("schedule: %+v", s)
}
f = withZFS(t)
got, _ = f.machine().Scrubs(context.Background(), false)
if a := got.(ScrubAnswer); a.Schedule != nil || !strings.Contains(strings.Join(a.Said, " "), "no scrub schedule") {
t.Errorf("no config: %+v", a)
}
}
func TestTheScheduleWarnsOfAnOldScrubAFailedRunAndAStoppedTimer(t *testing.T) {
f := scheduled(t)
h := f.machine().Health(context.Background())
if !h.Healthy {
t.Fatalf("%+v", h)
}
w := strings.Join(h.Warnings, " ")
// tank's last scrub finished 2026-09-14, 25 days before the fake clock: past weekly plus its margin.
if !strings.Contains(w, "Pool tank was last scrubbed 25 days ago, more than its weekly schedule allows (9 days)") ||
!strings.Contains(w, "The scheduled scrub of tank due 2026-10-05 00:00 has not started") {
t.Errorf("warnings: %v", h.Warnings)
}
f.on(line("systemctl", ServiceShowArgv(RunService)),
"ActiveState=failed\nResult=exit-code\nExecMainStartTimestamp=@1791237600\nExecMainExitTimestamp=@1791237601\n")
f.on(line("systemctl", TimerShowArgv(RunTimer)),
"ActiveState=inactive\nUnitFileState=disabled\nNextElapseUSecRealtime=\nLastTriggerUSec=@1791237600\n")
w = strings.Join(f.machine().Health(context.Background()).Warnings, " ")
for _, want := range []string{"The last scheduled scrub of tank failed (exit-code)", "The scrub schedule's timer zfs-mesh-run.timer is inactive and disabled"} {
if !strings.Contains(w, want) {
t.Errorf("lacks %q: %s", want, w)
}
}
}
func TestTheScrubActsAreFixedCommandsWithWhyReadBackAndRecorded(t *testing.T) {
for act, cmd := range map[string]string{
"start": "sudo -n zpool scrub -- tank",
"pause": "sudo -n zpool scrub -p -- tank",
"stop": "sudo -n zpool scrub -s -- tank",
} {
f := withZFS(t)
f.on(cmd, "")
a, err := f.machine().ScrubAct(context.Background(), act, "tank", "the operator asked")
if err != nil {
t.Fatalf("%s: %v", act, err)
}
if strings.Join(a.Ran, " ") != cmd || a.After == "" || !a.Done {
t.Errorf("%s: %+v", act, a)
}
if len(f.emitted) != 1 || f.emitted[0]["_type"] != ScrubEvent || f.emitted[0]["act"] != act || f.emitted[0]["why"] != "the operator asked" {
t.Errorf("%s recorded: %+v", act, f.emitted)
}
}
f := withZFS(t)
for _, c := range []struct{ act, pool, why, says string }{
{"start", "nope", "why", "no pool"},
{"start", "tank", "", "say why"},
{"start", "-s", "why", "no pool"},
{"destroy", "tank", "why", "not an act"},
} {
if _, err := f.machine().ScrubAct(context.Background(), c.act, c.pool, c.why); err == nil || !strings.Contains(err.Error(), c.says) {
t.Errorf("%+v: %v", c, err)
}
}
if n := len(ran(f, "sudo")); n != 0 {
t.Errorf("a refusal ran %v", f.calls)
}
f.answers["sudo -n zpool scrub -- tank"] = Ran{Code: -1, TimedOut: true}
if _, err := f.machine().ScrubAct(context.Background(), "start", "tank", "why"); err == nil || !strings.Contains(err.Error(), "outcome unknown; check zfs_scrub") {
t.Errorf("timed out: %v", err)
}
}
// Third review: the scheduled run's state is root's alone — a folder of its own, 0700 — so the operator account, or anything it runs, cannot make the run write elsewhere.
func TestTheServedSlotIsWrittenOnlyIntoAPrivateFolderAndNeverThroughALink(t *testing.T) {
dir := filepath.Join(t.TempDir(), "zfs-scrub")
path := filepath.Join(dir, "scrub-served.json")
if err := writeServed(path, map[string]int64{"tank": 1}); err != nil {
t.Fatal(err)
}
if fi, err := os.Lstat(dir); err != nil || fi.Mode().Perm() != 0o700 || !fi.IsDir() {
t.Errorf("the folder: %v %v", fi.Mode(), err)
}
// A folder open to others: refused. (The refusal of a link is not run here: no test of this module makes a
// symbolic link, the mesh's hard rule.)
open := filepath.Join(t.TempDir(), "open")
os.Mkdir(open, 0o755)
if err := writeServed(filepath.Join(open, "scrub-served.json"), map[string]int64{"tank": 3}); err == nil {
t.Errorf("written into a folder others can read")
}
if filepath.Dir(ServedFile) == filepath.Dir(ErrorsFile) {
t.Errorf("the run's state shares the account's folder")
}
}
+31 -116
View File
@@ -15,21 +15,19 @@ import (
// The scrub schedule the module owns.
//
// The mesh writes ScrubConfig from the assignment's settings. The node-engine runs the module's
// scheduled process, `zfs-tools scrub-run`, from its own timer (ScrubTimer) every quarter of an hour, and
// the run decides from the configuration whether a scrub is due: the last scheduled time not after now
// (the slot) not yet served. Due, it starts the pool's scrub, or waits for one already running, as
// upstream's zfs-scrub@.service does; it never pauses or stops one, and nothing stops the unit to pause
// it (the node-engine writes no ExecStop). So no setting value ever reaches a unit file: the day and the
// time are read and checked here, and an invalid one is refused before anything runs.
// The node-engine writes ConfigFile from the config values. The node-engine runs the module's scheduled
// process, `zfs-tools run`, from its own timer (RunTimer) every quarter of an hour, and the run decides from the
// config whether a scrub is due: the last scheduled time not after now (the slot) not yet served. Due, it starts
// the pool's scrub and returns; it never pauses or stops one and never waits for one (run.go). The day and the
// time are read and checked by the parse (config.go), so no setting value ever reaches a unit file.
//
// The mesh holds the distribution's own scrub timers for the pool stopped and disabled, so one schedule
// scrubs it; the node-engine gives them back running and enabled when the module goes (novox/hq ADR 0269).
// ScrubTimer and ScrubService are the node-engine's units for the scheduled run (the process's name).
// RunTimer and RunService are the node-engine's units for the scheduled run (the process's name).
const (
ScrubTimer = "zfs-mesh-scrub.timer"
ScrubService = "zfs-mesh-scrub.service"
RunTimer = "zfs-mesh-run.timer"
RunService = "zfs-mesh-run.service"
)
// ServedFile is where the scheduled run keeps the last slot it served, per pool: in a folder of root's
@@ -42,9 +40,6 @@ const ServedFile = "/var/lib/zfs-scrub/scrub-served.json"
// fires every quarter of an hour.
const MissedAfter = time.Hour
// ScrubConfig is the schedule as the mesh wrote it from the settings.
const ScrubConfig = "/etc/zfs-tools/scrub.conf"
// Margins after a cadence before a last scrub is said to be late: a weekly run that waited for a busy
// pool, a monthly run on the first given weekday of a month (28 or 35 days apart).
var cadenceAllows = map[string]time.Duration{
@@ -81,33 +76,22 @@ var (
clock = regexp.MustCompile(`^([01][0-9]|2[0-3]):[0-5][0-9]$`)
)
// ParseScrubConfig reads the schedule: pool, cadence (weekly or monthly), day (Mon … Sun) and time (HH:MM).
func ParseScrubConfig(text string) (*Schedule, error) {
v := map[string]string{}
for _, l := range strings.Split(text, "\n") {
l = strings.TrimSpace(l)
if l == "" || strings.HasPrefix(l, "#") {
continue
}
k, val, ok := strings.Cut(l, "=")
if !ok {
return nil, fmt.Errorf("%s: %q is not key=value", ScrubConfig, l)
}
v[strings.TrimSpace(k)] = strings.TrimSpace(val)
// ScheduleOf is the scrub schedule the config holds: the pool, and the cadence, day and time. nil when the
// values are not all set (the module then has no scrub schedule, §11: not managed until set).
func ScheduleOf(c *Config) *Schedule {
if c.Pool == "" || c.ScrubCadence == "" || c.ScrubDay == "" || c.ScrubTime == "" {
return nil
}
s := &Schedule{Pool: v["pool"], Cadence: v["cadence"], Day: v["day"], Time: v["time"]}
switch {
case !poolName.MatchString(s.Pool):
return nil, fmt.Errorf("the scrub-pool setting %q is not a pool's name", s.Pool)
case cadenceAllows[s.Cadence] == 0:
return nil, fmt.Errorf("the scrub-cadence setting is %q; it is weekly or monthly", s.Cadence)
case !weekday.MatchString(s.Day):
return nil, fmt.Errorf("the scrub-day setting is %q; it is Mon, Tue, Wed, Thu, Fri, Sat or Sun", s.Day)
case !clock.MatchString(s.Time):
return nil, fmt.Errorf("the scrub-time setting is %q; it is HH:MM, 00:00 to 23:59", s.Time)
return &Schedule{Pool: c.Pool, Cadence: c.ScrubCadence, Day: c.ScrubDay, Time: c.ScrubTime, Timer: RunTimer, Service: RunService}
}
// TrimScheduleOf is the trim schedule: nil while trim-schedule is not weekly or monthly, or its day or time is
// not set.
func TrimScheduleOf(c *Config) *Schedule {
if c.Pool == "" || (c.TrimSchedule != "weekly" && c.TrimSchedule != "monthly") || c.TrimDay == "" || c.TrimTime == "" {
return nil
}
s.Timer, s.Service = ScrubTimer, ScrubService
return s, nil
return &Schedule{Pool: c.Pool, Cadence: c.TrimSchedule, Day: c.TrimDay, Time: c.TrimTime, Timer: RunTimer, Service: RunService}
}
var weekdays = map[string]time.Weekday{"Sun": time.Sunday, "Mon": time.Monday, "Tue": time.Tuesday,
@@ -150,13 +134,6 @@ func (s *Schedule) NextSlot(now time.Time) time.Time {
return time.Time{}
}
// WaitArgv waits for a pool's running scrub to end.
func WaitArgv(pool string) []string { return []string{"wait", "-t", "scrub", "--", pool} }
// WaitBound is how long the scheduled run waits for a scrub: past it the run ends and leaves the scrub
// running; a scrub never pauses for it.
const WaitBound = 7 * 24 * time.Hour
// readServed is the last slot served per pool; none when the file is not there or not readable.
func (m *Machine) readServed(path string) map[string]int64 {
out := map[string]int64{}
@@ -220,72 +197,6 @@ func startedSince(p *PoolStatus, slot time.Time) bool {
return sc != nil && sc.Started != nil && !sc.Started.Before(slot)
}
// ScrubRun is the scheduled run: a due scrub started, or one running waited for. It answers what it did in
// a line, and fails on an invalid schedule, which it never acts on.
func (m *Machine) ScrubRun(ctx context.Context, servedPath string) (string, error) {
raw, err := m.ReadFile(ScrubConfig)
if err != nil {
return "", fmt.Errorf("%s could not be read: %v", ScrubConfig, err)
}
s, err := ParseScrubConfig(string(raw))
if err != nil {
return "", fmt.Errorf("refused, nothing run: %v", err)
}
now := m.Now()
slot := s.Slot(now)
served := m.readServed(servedPath)
if served[s.Pool] >= slot.Unix() {
return fmt.Sprintf("the scrub of %s scheduled %s was served", s.Pool, slot.Format("2006-01-02 15:04")), nil
}
st, err := m.Status(ctx)
if err != nil {
return "", err
}
p := statusOf(st, s.Pool)
if p == nil {
return "", fmt.Errorf("the scheduled pool %s is not imported here", s.Pool)
}
mark := func() error {
served[s.Pool] = slot.Unix()
return writeServed(servedPath, served)
}
if startedSince(p, slot) && p.Scrub().State != "running" {
return fmt.Sprintf("the scrub of %s scheduled %s ran already", s.Pool, slot.Format("2006-01-02 15:04")), mark()
}
for _, r := range p.Resilvers() {
if r.State == "running" {
return fmt.Sprintf("a resilver of %s runs; the scrub scheduled %s waits for the next run after it", s.Pool,
slot.Format("2006-01-02 15:04")), nil
}
}
started := false
if sc := p.Scrub(); sc == nil || sc.State != "running" {
_, err := m.AsRoot(ctx, "zpool", ScrubArgv("start", s.Pool)...)
switch {
case err == nil:
started = true
case strings.Contains(err.Error(), "currently scrubbing"):
// One began since status was read: it serves the slot, and is waited for like any other.
default:
return "", fmt.Errorf("the scheduled scrub of %s could not start: %v", s.Pool, err)
}
}
if err := mark(); err != nil {
return "", fmt.Errorf("the scrub of %s runs, and the served slot could not be kept: %v", s.Pool, err)
}
wctx, cancel := context.WithTimeout(ctx, WaitBound)
defer cancel()
_, werr := m.AsRoot(wctx, "zpool", WaitArgv(s.Pool)...)
what := "waited for the scrub of " + s.Pool + " already running"
if started {
what = "started the scrub of " + s.Pool + " scheduled " + slot.Format("2006-01-02 15:04") + ", and waited for it"
}
if werr != nil {
return what + "; the wait ended without its end (" + werr.Error() + "), and the scrub goes on", nil
}
return what + " to finish", nil
}
// TimerShowArgv asks the service manager what a timer says: whether it runs and is enabled, when it
// fires next and when it last did, as Unix times.
func TimerShowArgv(unit string) []string {
@@ -320,15 +231,19 @@ func unixStamp(s string) *time.Time {
}
// ScheduleNow is the schedule with what the service manager says of it; nil when the module has no
// schedule on this machine (no configuration written).
// schedule on this machine (no config file, or no cadence, day and time set).
func (m *Machine) ScheduleNow(ctx context.Context) (*Schedule, error) {
raw, err := m.ReadFile(ScrubConfig)
raw, err := m.ReadFile(ConfigFile)
if err != nil {
return nil, nil
}
s, err := ParseScrubConfig(string(raw))
if err != nil {
return nil, err
c, refused := ParseConfig(raw, m.memMiB())
if len(refused) > 0 {
return nil, fmt.Errorf("%s refused %s %s", ConfigFile, refused[0].Key, refused[0].Reason)
}
s := ScheduleOf(c)
if s == nil {
return nil, nil
}
if out, err := m.ask(ctx, "systemctl", TimerShowArgv(s.Timer)...); err != nil {
s.Note = "the service manager did not say what the timer is: " + err.Error()
+1 -1
View File
@@ -141,7 +141,7 @@ func (m *Machine) Scrubs(ctx context.Context, history bool) (any, error) {
case serr != nil:
a.Said = append(a.Said, "The scrub schedule's settings are not valid: "+serr.Error()+".")
case sched == nil:
a.Said = append(a.Said, "The module has no scrub schedule on this machine (no "+ScrubConfig+").")
a.Said = append(a.Said, "The module has no scrub schedule on this machine (no cadence, day and time set in "+ConfigFile+").")
default:
a.Schedule = sched
line := fmt.Sprintf("%s scrub of %s, %s at %s (monthly: the first %s of the month), decided by the run %s fires "+
-206
View File
@@ -1,206 +0,0 @@
package main
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
// The scheduled run (`zfs-tools scrub-run`, second review of mesh-catalog #147): the node-engine's timer
// fires it every quarter of an hour, and it decides from the configuration whether a scrub is due. It
// starts one, or waits for one already running, as upstream's zfs-scrub@.service does, and never pauses or
// stops a scrub.
func monthlyConf(pool string) string {
return "pool=" + pool + "\ncadence=monthly\nday=Mon\ntime=00:00\n"
}
func TestTheSlotIsTheLastScheduledTimeNotAfterNow(t *testing.T) {
loc := time.Local
at := func(y int, m time.Month, d, h, min int) time.Time { return time.Date(y, m, d, h, min, 0, 0, loc) }
cases := []struct {
conf string
now time.Time
want time.Time
}{
{"pool=tank\ncadence=weekly\nday=Mon\ntime=00:00\n", at(2026, 10, 9, 12, 0), at(2026, 10, 5, 0, 0)},
{"pool=tank\ncadence=weekly\nday=Fri\ntime=13:30\n", at(2026, 10, 9, 12, 0), at(2026, 10, 2, 13, 30)},
{"pool=tank\ncadence=weekly\nday=Fri\ntime=11:30\n", at(2026, 10, 9, 12, 0), at(2026, 10, 9, 11, 30)},
{monthlyConf("tank"), at(2026, 10, 9, 12, 0), at(2026, 10, 5, 0, 0)},
{monthlyConf("tank"), at(2026, 10, 4, 23, 0), at(2026, 9, 7, 0, 0)},
{"pool=tank\ncadence=monthly\nday=Thu\ntime=06:00\n", at(2026, 10, 1, 6, 0), at(2026, 10, 1, 6, 0)},
}
for _, c := range cases {
s, err := ParseScrubConfig(c.conf)
if err != nil {
t.Fatal(err)
}
if got := s.Slot(c.now); !got.Equal(c.want) {
t.Errorf("%q at %s: slot %s, want %s", c.conf, c.now, got, c.want)
}
if next := s.NextSlot(c.now); !next.After(c.now) || !s.Slot(next).Equal(next) {
t.Errorf("%q at %s: next slot %s", c.conf, c.now, next)
}
}
}
// root runs the scheduled run: no sudo.
func rootFake(t *testing.T, conf string) (*fake, string) {
f := withZFS(t)
f.uid = 0
f.files[ScrubConfig] = conf
dir := filepath.Join(t.TempDir(), "zfs-scrub")
return f, filepath.Join(dir, "scrub-served.json")
}
func TestADueScrubIsStartedThenWaitedForAndMarkedServed(t *testing.T) {
f, served := rootFake(t, monthlyConf("tank"))
f.on("zpool scrub -- tank", "")
f.on("zpool wait -t scrub -- tank", "")
said, err := f.machine().ScrubRun(context.Background(), served)
if err != nil {
t.Fatal(err)
}
if got := strings.Join(ran(f, "zpool scrub"), "|") + "/" + strings.Join(ran(f, "zpool wait"), "|"); got != "zpool scrub -- tank/zpool wait -t scrub -- tank" {
t.Errorf("ran %q", got)
}
if !strings.Contains(said, "started") {
t.Errorf("said %q", said)
}
// Served: the next fire within the same slot runs nothing.
f.calls = nil
if _, err := f.machine().ScrubRun(context.Background(), served); err != nil {
t.Fatal(err)
}
if n := len(ran(f, "zpool scrub")) + len(ran(f, "zpool wait")); n != 0 {
t.Errorf("a served slot ran %v", f.calls)
}
}
func TestAScrubInProgressIsWaitedForNeverPausedOrStarted(t *testing.T) {
f, served := rootFake(t, monthlyConf("tank"))
f.on(line("zpool", StatusArgv()), fixture(t, "zpool-status-scrubbing.txt"))
f.on(line("zpool", PoolsArgv()), "tank\t1000\t100\t900\t10\t1\tONLINE\t1.00\n")
f.on("zpool wait -t scrub -- tank", "")
if _, err := f.machine().ScrubRun(context.Background(), served); err != nil {
t.Fatal(err)
}
if strings.Join(f.calls, "|") == "" || len(ran(f, "zpool scrub")) != 0 || len(ran(f, "zpool wait -t scrub -- tank")) != 1 {
t.Errorf("ran %v", f.calls)
}
}
func TestAStartRefusedBecauseAScrubJustBeganIsWaitedForNotPaused(t *testing.T) {
f, served := rootFake(t, monthlyConf("tank"))
f.answers["zpool scrub -- tank"] = Ran{Code: 1, Err: "cannot scrub tank: currently scrubbing; use 'zpool scrub -s' to cancel the current scrub"}
f.on("zpool wait -t scrub -- tank", "")
if _, err := f.machine().ScrubRun(context.Background(), served); err != nil {
t.Fatal(err)
}
for _, c := range f.calls {
if strings.Contains(c, "scrub -p") || strings.Contains(c, "scrub -s") {
t.Errorf("the run paused or stopped a scrub: %s", c)
}
}
if len(ran(f, "zpool wait -t scrub -- tank")) != 1 {
t.Errorf("not waited for: %v", f.calls)
}
}
func TestAResilverDefersTheScrubWithoutServingTheSlot(t *testing.T) {
f, served := rootFake(t, monthlyConf("tank"))
f.on(line("zpool", StatusArgv()), fixture(t, "zpool-status-degraded.txt"))
f.on(line("zpool", PoolsArgv()), "tank\t1000\t100\t900\t10\t1\tDEGRADED\t1.00\n")
said, err := f.machine().ScrubRun(context.Background(), served)
if err != nil {
t.Fatal(err)
}
if len(ran(f, "zpool scrub")) != 0 || !strings.Contains(said, "resilver") {
t.Errorf("ran %v, said %q", f.calls, said)
}
if _, err := os.Stat(served); err == nil {
t.Errorf("the slot was marked served while the scrub waits for the resilver")
}
}
func TestAScrubAlreadyStartedInTheSlotServesItWithoutAStateFile(t *testing.T) {
// tank's scrub started 2026-09-13 00:24 (finished 09-14); a weekly Sunday-00:00 slot of 09-13 is served
// by it, so a run on 09-15 starts nothing.
f, served := rootFake(t, "pool=tank\ncadence=weekly\nday=Sun\ntime=00:00\n")
f.now = time.Date(2026, 9, 15, 12, 0, 0, 0, time.Local)
if _, err := f.machine().ScrubRun(context.Background(), served); err != nil {
t.Fatal(err)
}
if n := len(ran(f, "zpool scrub")) + len(ran(f, "zpool wait")); n != 0 {
t.Errorf("ran %v", f.calls)
}
}
func TestAnInvalidScheduleRunsNothingAndFails(t *testing.T) {
for _, conf := range []string{
"pool=tank\ncadence=monthly\nday=Mon\ntime=*:*\n",
"pool=tank\ncadence=monthly\nday=Mon..Fri\ntime=00:00\n",
"pool=tank\ncadence=monthly\nday=*\ntime=00:00\n",
"pool=tank\ncadence=hourly\nday=Mon\ntime=00:00\n",
"pool=tank\ncadence=monthly\nday=Mon\ntime=0:00\n",
} {
f, served := rootFake(t, conf)
if _, err := f.machine().ScrubRun(context.Background(), served); err == nil {
t.Errorf("%q: not refused", conf)
}
if len(ran(f, "zpool scrub"))+len(ran(f, "zpool wait")) != 0 {
t.Errorf("%q ran %v", conf, f.calls)
}
}
}
func TestAScheduledScrubThatDidNotStartIsAWarning(t *testing.T) {
f := withZFS(t)
f.files[ScrubConfig] = "pool=tank\ncadence=weekly\nday=Mon\ntime=00:00\n"
f.on(line("systemctl", TimerShowArgv(ScrubTimer)), "ActiveState=active\nUnitFileState=enabled\nNextElapseUSecRealtime=@1791842400\nLastTriggerUSec=@1791237600\n")
h := f.machine().Health(context.Background())
if !strings.Contains(strings.Join(h.Warnings, " "), "The scheduled scrub of tank due 2026-10-05 00:00 has not started") {
t.Errorf("warnings: %v", h.Warnings)
}
}
// Third review: the scheduled run's state is root's alone — a folder of its own, 0700, never through a
// link — so the operator account, or anything it runs, cannot make the run write elsewhere.
func TestTheServedSlotIsWrittenOnlyIntoAPrivateFolderAndNeverThroughALink(t *testing.T) {
dir := filepath.Join(t.TempDir(), "zfs-scrub")
path := filepath.Join(dir, "scrub-served.json")
if err := writeServed(path, map[string]int64{"tank": 1}); err != nil {
t.Fatal(err)
}
if fi, err := os.Lstat(dir); err != nil || fi.Mode().Perm() != 0o700 || !fi.IsDir() {
t.Errorf("the folder: %v %v", fi.Mode(), err)
}
// The file replaced by a link to somewhere else: refused, and the other place untouched.
victim := filepath.Join(t.TempDir(), "victim")
os.WriteFile(victim, []byte("keep"), 0o644)
os.Remove(path)
os.Symlink(victim, path)
if err := writeServed(path, map[string]int64{"tank": 2}); err == nil {
t.Errorf("written through a link")
}
if b, _ := os.ReadFile(victim); string(b) != "keep" {
t.Errorf("the link's target was changed: %q", b)
}
// The folder itself a link, or open to others: refused.
linked := filepath.Join(t.TempDir(), "linked")
os.Symlink(t.TempDir(), linked)
if err := writeServed(filepath.Join(linked, "scrub-served.json"), map[string]int64{"tank": 3}); err == nil {
t.Errorf("written into a linked folder")
}
open := filepath.Join(t.TempDir(), "open")
os.Mkdir(open, 0o755)
if err := writeServed(filepath.Join(open, "scrub-served.json"), map[string]int64{"tank": 3}); err == nil {
t.Errorf("written into a folder others can read")
}
if filepath.Dir(ServedFile) == filepath.Dir(ErrorsFile) {
t.Errorf("the run's state shares the account's folder")
}
}
@@ -1,119 +0,0 @@
package main
import (
"context"
"strings"
"testing"
"time"
)
const scrubConf = "pool=tank\ncadence=weekly\nday=Mon\ntime=00:00\n"
// scheduled is the healthy fixtures with the module's scrub schedule for tank, its timer waiting and its
// last run a success.
func scheduled(t *testing.T) *fake {
f := withZFS(t)
f.files[ScrubConfig] = scrubConf
f.on(line("systemctl", TimerShowArgv(ScrubTimer)),
"ActiveState=active\nUnitFileState=enabled\nNextElapseUSecRealtime=@1791842400\nLastTriggerUSec=@1791237600\n")
f.on(line("systemctl", ServiceShowArgv(ScrubService)),
"ActiveState=inactive\nResult=success\nExecMainStartTimestamp=@1791237600\nExecMainExitTimestamp=@1791331200\n")
return f
}
func TestTheScheduleIsTheConfigAndWhatTheTimerSays(t *testing.T) {
f := scheduled(t)
got, err := f.machine().Scrubs(context.Background(), false)
if err != nil {
t.Fatal(err)
}
a := got.(ScrubAnswer)
s := a.Schedule
if s == nil || s.Pool != "tank" || s.Cadence != "weekly" || s.Timer != "zfs-mesh-scrub.timer" || s.TimerState != "active" ||
s.Enabled != "enabled" || s.Next == nil || !s.Next.Equal(time.Date(2026, 10, 12, 0, 0, 0, 0, time.Local)) || s.LastResult != "success" {
t.Fatalf("schedule: %+v", s)
}
if want := s.NextSlot(f.now).Format("2006-01-02 15:04"); !strings.Contains(strings.Join(a.Said, "\n"), "next scheduled scrub of tank: "+want) || want != "2026-10-12 00:00" {
t.Errorf("said: %v", a.Said)
}
}
func TestAScheduleThatIsNotSetIsSaidAndAnInvalidOneRefused(t *testing.T) {
f := withZFS(t)
got, _ := f.machine().Scrubs(context.Background(), false)
if a := got.(ScrubAnswer); a.Schedule != nil || !strings.Contains(strings.Join(a.Said, " "), "no scrub schedule") {
t.Errorf("no config: %+v", a)
}
for _, conf := range []string{
"pool=tank\ncadence=daily\nday=Mon\ntime=00:00\n",
"pool=tank\ncadence=weekly\nday=Monday\ntime=00:00\n",
"pool=tank\ncadence=weekly\nday=Mon\ntime=25:00\n",
"pool=tank x\ncadence=weekly\nday=Mon\ntime=00:00\n",
} {
if _, err := ParseScrubConfig(conf); err == nil {
t.Errorf("accepted %q", conf)
}
}
}
func TestTheScheduleWarnsOfAnOldScrubAFailedRunAndAStoppedTimer(t *testing.T) {
f := scheduled(t)
h := f.machine().Health(context.Background())
if !h.Healthy {
t.Fatalf("%+v", h)
}
w := strings.Join(h.Warnings, " ")
// tank's last scrub finished 2026-09-14, 25 days before the fake clock: past weekly plus its margin.
if !strings.Contains(w, "Pool tank was last scrubbed 25 days ago, more than its weekly schedule allows (9 days)") {
t.Errorf("warnings: %v", h.Warnings)
}
f.on(line("systemctl", ServiceShowArgv(ScrubService)),
"ActiveState=failed\nResult=exit-code\nExecMainStartTimestamp=@1791237600\nExecMainExitTimestamp=@1791237601\n")
f.on(line("systemctl", TimerShowArgv(ScrubTimer)),
"ActiveState=inactive\nUnitFileState=disabled\nNextElapseUSecRealtime=\nLastTriggerUSec=@1791237600\n")
w = strings.Join(f.machine().Health(context.Background()).Warnings, " ")
for _, want := range []string{"The last scheduled scrub of tank failed (exit-code)", "The scrub schedule's timer zfs-mesh-scrub.timer is inactive and disabled"} {
if !strings.Contains(w, want) {
t.Errorf("lacks %q: %s", want, w)
}
}
}
func TestTheScrubActsAreFixedCommandsWithWhyReadBackAndRecorded(t *testing.T) {
for act, cmd := range map[string]string{
"start": "sudo -n zpool scrub -- tank",
"pause": "sudo -n zpool scrub -p -- tank",
"stop": "sudo -n zpool scrub -s -- tank",
} {
f := withZFS(t)
f.on(cmd, "")
a, err := f.machine().ScrubAct(context.Background(), act, "tank", "the operator asked")
if err != nil {
t.Fatalf("%s: %v", act, err)
}
if strings.Join(a.Ran, " ") != cmd || a.After == "" || !a.Done {
t.Errorf("%s: %+v", act, a)
}
if len(f.emitted) != 1 || f.emitted[0]["_type"] != ScrubEvent || f.emitted[0]["act"] != act || f.emitted[0]["why"] != "the operator asked" {
t.Errorf("%s recorded: %+v", act, f.emitted)
}
}
f := withZFS(t)
for _, c := range []struct{ act, pool, why, says string }{
{"start", "nope", "why", "no pool"},
{"start", "tank", "", "say why"},
{"start", "-s", "why", "no pool"},
{"destroy", "tank", "why", "not an act"},
} {
if _, err := f.machine().ScrubAct(context.Background(), c.act, c.pool, c.why); err == nil || !strings.Contains(err.Error(), c.says) {
t.Errorf("%+v: %v", c, err)
}
}
if n := len(ran(f, "sudo")); n != 0 {
t.Errorf("a refusal ran %v", f.calls)
}
f.answers["sudo -n zpool scrub -- tank"] = Ran{Code: -1, TimedOut: true}
if _, err := f.machine().ScrubAct(context.Background(), "start", "tank", "why"); err == nil || !strings.Contains(err.Error(), "outcome unknown; check zfs_scrub") {
t.Errorf("timed out: %v", err)
}
}
+3 -3
View File
@@ -187,13 +187,13 @@
"unknown": "program"
},
{
"id": "scrub-run",
"id": "run",
"type": "process",
"name": "zfs-mesh-scrub",
"name": "zfs-mesh-run",
"artifact": "tools",
"run": [
"./zfs-tools",
"scrub-run"
"run"
],
"schedule": "0,15,30,45 * * * *"
},