From 06d83912cdbdae40e8553368899164203da6f678 Mon Sep 17 00:00:00 2001 From: jochens Date: Fri, 13 Nov 2026 00:38:17 +0100 Subject: [PATCH] Z10 zfs-run: the scheduled run takes the lock, starts a due scrub and trim without waiting, and acts on hand acts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 42 zfs-owns-zfs §20, §24, A-17 --- modules/zfs/cmd/zfs-tools/apply.go | 5 + modules/zfs/cmd/zfs-tools/main.go | 16 +- modules/zfs/cmd/zfs-tools/manifest_test.go | 26 +- modules/zfs/cmd/zfs-tools/run.go | 168 +++++++++++ modules/zfs/cmd/zfs-tools/run_test.go | 269 ++++++++++++++++++ modules/zfs/cmd/zfs-tools/schedule.go | 147 ++-------- modules/zfs/cmd/zfs-tools/scrub.go | 2 +- modules/zfs/cmd/zfs-tools/scrubrun_test.go | 206 -------------- .../zfs/cmd/zfs-tools/scrubschedule_test.go | 119 -------- modules/zfs/module.json | 6 +- 10 files changed, 502 insertions(+), 462 deletions(-) create mode 100644 modules/zfs/cmd/zfs-tools/run.go create mode 100644 modules/zfs/cmd/zfs-tools/run_test.go delete mode 100644 modules/zfs/cmd/zfs-tools/scrubrun_test.go delete mode 100644 modules/zfs/cmd/zfs-tools/scrubschedule_test.go diff --git a/modules/zfs/cmd/zfs-tools/apply.go b/modules/zfs/cmd/zfs-tools/apply.go index beabb0ebd..bcd5ed579 100644 --- a/modules/zfs/cmd/zfs-tools/apply.go +++ b/modules/zfs/cmd/zfs-tools/apply.go @@ -149,6 +149,11 @@ func (m *Machine) Apply(ctx context.Context) (string, error) { return "", fmt.Errorf("the apply's lock %s: %v", ApplyLock, err) } defer release() + return m.applyLocked(ctx) +} + +// applyLocked is the apply under a lock already held (the scheduled run's hand act takes the same lock). +func (m *Machine) applyLocked(ctx context.Context) (string, error) { text, err := m.ReadFile(ConfigFile) if err != nil { return "", fmt.Errorf("the config file %s cannot be read: %v", ConfigFile, err) diff --git a/modules/zfs/cmd/zfs-tools/main.go b/modules/zfs/cmd/zfs-tools/main.go index 18d079383..1056e65e3 100644 --- a/modules/zfs/cmd/zfs-tools/main.go +++ b/modules/zfs/cmd/zfs-tools/main.go @@ -6,8 +6,8 @@ // speaks MCP to over stdio (novox/hq ADR 0188, ADR 0193), as the operator account. With `watch` and // `events` it is one of the module's two long-running processes, run by the node-engine: the watch, which // the health check zfs_health stands on (watch.go), and the event follower, as root, judged by its unit -// (follow.go). With `scrub-run` it is the scheduled scrub, run as root by the node-engine's timer -// (schedule.go). +// (follow.go). With `run` it is the scheduled run, as root by the node-engine's timer (run.go), and with `apply` the +// apply that converges the node to its config file (apply.go). package main import ( @@ -27,8 +27,14 @@ func main() { switch os.Args[1] { case "watch": (&Watch{M: Here(), Out: os.Stdout}).Loop(ctx) - case "scrub-run": - said, err := Here().ScrubRun(ctx, ServedFile) + case "run": + m := Here() + closeBus := m.JoinBus() + said, err := m.RunScheduled(ctx, ServedFile) + closeBus() + for _, w := range m.Warnings { + fmt.Fprintln(os.Stderr, "warning:", w) + } if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) @@ -54,7 +60,7 @@ func main() { } default: fmt.Fprintf(os.Stderr, "zfs-tools: %q is not a way to run; with no argument it serves its tools, with watch it "+ - "watches the pools, with events it follows ZFS's events, with scrub-run it runs the scheduled scrub, with apply it converges the node to its config file\n", os.Args[1]) + "watches the pools, with events it follows ZFS's events, with run it runs the scheduled scrub, trim and snapshots, with apply it converges the node to its config file\n", os.Args[1]) os.Exit(2) } return diff --git a/modules/zfs/cmd/zfs-tools/manifest_test.go b/modules/zfs/cmd/zfs-tools/manifest_test.go index 93ca39be8..079bbc266 100644 --- a/modules/zfs/cmd/zfs-tools/manifest_test.go +++ b/modules/zfs/cmd/zfs-tools/manifest_test.go @@ -119,10 +119,10 @@ func TestExactlyOneScrubScheduleAndNoSettingInAUnitFile(t *testing.T) { byID[r["id"].(string)] = r order[r["id"].(string)] = i } - run := byID["scrub-run"] + run := byID["run"] args, _ := run["run"].([]any) - if run["type"] != "process" || run["name"] != "zfs-mesh-scrub" || run["schedule"] != "0,15,30,45 * * * *" || run["user"] != nil || - len(args) != 2 || args[1] != "scrub-run" { + if run["type"] != "process" || run["name"] != "zfs-mesh-run" || run["schedule"] != "0,15,30,45 * * * *" || run["user"] != nil || + len(args) != 2 || args[1] != "run" { t.Errorf("the scheduled run: %v", run) } // The node-engine writes a cron step ("*/15") as a calendar systemd refuses ("*:*/15"), and the timer never @@ -131,15 +131,15 @@ func TestExactlyOneScrubScheduleAndNoSettingInAUnitFile(t *testing.T) { if s, _ := run["schedule"].(string); strings.Contains(s, "/") { t.Errorf("the scheduled run's schedule %q uses a step, which the node-engine writes as a calendar systemd refuses", s) } - if ScrubTimer != run["name"].(string)+".timer" || ScrubService != run["name"].(string)+".service" { - t.Errorf("the tools read %s and %s; the node-engine names them after %v", ScrubTimer, ScrubService, run["name"]) + if RunTimer != run["name"].(string)+".timer" || RunService != run["name"].(string)+".service" { + t.Errorf("the tools read %s and %s; the node-engine names them after %v", RunTimer, RunService, run["name"]) } for _, c := range []string{"weekly", "monthly"} { d := byID["distribution-"+c] if d["unit"] != "zfs-scrub-"+c+"@${config:scrub-pool}.timer" || d["state"] != "stopped" || d["boot"] != "disabled" { t.Errorf("the distribution's %s timer: %v", c, d) } - if order["scrub-run"] > order["distribution-"+c] { + if order["run"] > order["distribution-"+c] { t.Errorf("the distribution's %s timer is stopped before the run is installed", c) } } @@ -188,7 +188,7 @@ func TestTheHomeServersComposedUnitsAndConfiguration(t *testing.T) { conf = fill.Replace(templateText(t, r)) } } - want := []string{"zfs-mesh-scrub.timer running/enabled (the node-engine's, every quarter of an hour)", + want := []string{"zfs-mesh-run.timer running/enabled (the node-engine's, every quarter of an hour)", "zfs-scrub-weekly@storage.timer stopped/disabled", "zfs-scrub-monthly@storage.timer stopped/disabled"} if strings.Join(units, "\n") != strings.Join(want, "\n") { t.Errorf("units:\n%s", strings.Join(units, "\n")) @@ -196,9 +196,11 @@ func TestTheHomeServersComposedUnitsAndConfiguration(t *testing.T) { for _, u := range units { t.Log(u) } - s, err := ParseScrubConfig(conf) - if err != nil || s.Pool != "storage" || s.Cadence != "monthly" { - t.Fatalf("configuration %q: %v", conf, err) + _ = conf + cfg, refused := ParseConfig([]byte(scrubConf("storage", "monthly", "Mon", "00:00")), 16000) + s := ScheduleOf(cfg) + if len(refused) != 0 || s == nil || s.Pool != "storage" || s.Cadence != "monthly" { + t.Fatalf("configuration: %v", refused) } t.Logf("next scrub of storage after 2026-10-09 12:00: %s", s.NextSlot(time.Date(2026, 10, 9, 12, 0, 0, 0, time.Local)).Format("2006-01-02 15:04")) if next := s.NextSlot(time.Date(2026, 10, 9, 12, 0, 0, 0, time.Local)); !next.Equal(time.Date(2026, 11, 2, 0, 0, 0, 0, time.Local)) { @@ -216,7 +218,7 @@ func TestTheBundleRunsOnlyReadingVerbsSnapshotAndScrub(t *testing.T) { "SnapshotArgv": {"zfs", "snapshot"}, "PoolsArgv": {"zpool", "list"}, "StatusArgv": {"zpool", "status"}, "IOArgv": {"zpool", "iostat"}, "EventsArgv": {"zpool", "events"}, "HistoryArgv": {"zpool", "history"}, "FollowArgv": {"zpool", "events"}, - "ScrubArgv": {"zpool", "scrub"}, "WaitArgv": {"zpool", "wait"}, + "ScrubArgv": {"zpool", "scrub"}, "TrimArgv": {"zpool", "trim"}, "TimerShowArgv": {"systemctl", "show"}, "ServiceShowArgv": {"systemctl", "show"}, "PoolGetArgv": {"zpool", "get"}, "PoolSetArgv": {"zpool", "set"}, "DatasetGetArgv": {"zfs", "get"}, "DatasetSetArgv": {"zfs", "set"}, "DatasetInheritArgv": {"zfs", "inherit"}, @@ -227,7 +229,7 @@ func TestTheBundleRunsOnlyReadingVerbsSnapshotAndScrub(t *testing.T) { verbs := map[string][]string{"DatasetsArgv": DatasetsArgv(), "SnapshotsArgv": SnapshotsArgv(), "PoolsArgv": PoolsArgv(), "StatusArgv": StatusArgv(), "IOArgv": IOArgv(3), "EventsArgv": EventsArgv(), "HistoryArgv": HistoryArgv("tank"), "PropertiesArgv": PropertiesArgv("tank"), "SnapshotArgv": SnapshotArgv("tank", "x"), "FollowArgv": FollowArgv(), - "TimerShowArgv": TimerShowArgv("x.timer"), "ServiceShowArgv": ServiceShowArgv("x.service"), "WaitArgv": WaitArgv("tank"), + "TimerShowArgv": TimerShowArgv("x.timer"), "ServiceShowArgv": ServiceShowArgv("x.service"), "TrimArgv": TrimArgv("tank"), "PoolGetArgv": PoolGetArgv("tank", "autotrim"), "PoolSetArgv": PoolSetArgv("tank", "autotrim", "on"), "DatasetGetArgv": DatasetGetArgv("tank", "atime"), "DatasetSetArgv": DatasetSetArgv("tank", "atime", "on"), "DatasetInheritArgv": DatasetInheritArgv("tank", "atime"), diff --git a/modules/zfs/cmd/zfs-tools/run.go b/modules/zfs/cmd/zfs-tools/run.go new file mode 100644 index 000000000..922cefdf9 --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/run.go @@ -0,0 +1,168 @@ +package main + +import ( + "context" + "fmt" + "sort" + "strings" +) + +// The scheduled run (design 42 zfs-owns-zfs §20, §24). +// +// `zfs-tools run` is the module's one scheduled run, root's, fired by the node-engine's timer every quarter of +// an hour. Each run takes the apply's lock; held, it does nothing and says `run.skipped` {reason: lock-held}. +// It does nothing while a scrub or resilver runs or before the pool is imported, saying that reason. +// Otherwise it acts on the hand acts waiting in the state `schedule`, starts the scrub when due and returns +// at once (the scrub's end is the event follower's to hear, §27), and starts a trim when trim-schedule is due. + +// The reasons of `run.skipped`. +const ( + SkipLockHeld = "lock-held" + SkipPoolNotImported = "pool-not-imported" + SkipScanRunning = "scrub-or-resilver-running" +) + +// KeyPaused is the key of the state `schedule` that holds the schedule's snapshots (§24). +const KeyPaused = "paused" + +// TrimArgv starts a trim of the pool. +func TrimArgv(pool string) []string { return []string{"trim", "--", pool} } + +// handActs are the acts waiting in the state `schedule`, each under its own key (A-17), but the pause. +func (m *Machine) handActs() []string { + if m.States == nil { + return nil + } + keys, err := m.States.Keys(StateSchedule) + if err != nil { + return nil + } + var acts []string + for _, k := range keys { + if k != KeyPaused { + acts = append(acts, k) + } + } + sort.Strings(acts) + return acts +} + +func (m *Machine) skip(reason string) string { + var failed []error + if err := m.say(EventRunSkipped, map[string]any{"reason": reason}); err != nil { + m.failure(&failed, "run.skipped", err) + } + return "skipped: " + reason +} + +// RunScheduled is the scheduled run. It answers what it did in a line; its error is a command that failed. +func (m *Machine) RunScheduled(ctx context.Context, servedPath string) (string, error) { + release, got, err := m.Lock(ApplyLock, false) + if err != nil { + return "", fmt.Errorf("the lock %s: %v", ApplyLock, err) + } + if !got { + return m.skip(SkipLockHeld), nil + } + defer release() + text, err := m.ReadFile(ConfigFile) + if err != nil { + return "", fmt.Errorf("%s could not be read: %v", ConfigFile, err) + } + c, refused := ParseConfig(text, m.memMiB()) + if len(refused) > 0 { + return "refused, nothing run: " + refused[0].Key + " " + refused[0].Reason, nil + } + var said []string + var failed []error + acts := m.handActs() + guard := "" + var p *PoolStatus + if c.Pool != "" { + wait, perr := m.poolWait(ctx, c.Pool) + switch { + case perr != nil: + return "", perr + case strings.Contains(wait, "to be imported"): + guard = SkipPoolNotImported + case wait != "": + guard = SkipScanRunning + } + if guard == "" { + st, err := m.Status(ctx) + if err != nil { + return "", err + } + p = statusOf(st, c.Pool) + } + } + for _, act := range acts { + switch { + case act == "apply": + s, err := m.applyLocked(ctx) + said = append(said, "apply: "+s) + if err != nil { + failed = append(failed, err) + } + case guard != "": + said = append(said, m.skip(guard)) + default: + m.Warnings = append(m.Warnings, "the hand act "+act+" is not one this run knows") + } + _ = m.States.Delete(StateSchedule, act) + } + if guard != "" || c.Pool == "" { + if guard != "" && len(acts) == 0 { + said = append(said, m.skip(guard)) + } + return strings.Join(said, "; "), joinErrors(failed) + } + served := m.readServed(servedPath) + now := m.Now() + for _, work := range []struct { + name, key, verb string + s *Schedule + start func() error + }{{"scrub", c.Pool, "start", ScheduleOf(c), func() error { + _, err := m.AsRoot(ctx, "zpool", ScrubArgv("start", c.Pool)...) + return err + }}, {"trim", c.Pool + "/trim", "start", TrimScheduleOf(c), func() error { + _, err := m.AsRoot(ctx, "zpool", TrimArgv(c.Pool)...) + return err + }}} { + if work.s == nil { + continue + } + slot := work.s.Slot(now) + if slot.IsZero() || served[work.key] >= slot.Unix() { + continue + } + if work.name == "scrub" && startedSince(p, slot) { + said = append(said, fmt.Sprintf("the scrub of %s scheduled %s ran already", c.Pool, slot.Format("2006-01-02 15:04"))) + } else if err := work.start(); err != nil && !strings.Contains(err.Error(), "currently") { + failed = append(failed, fmt.Errorf("the scheduled %s of %s could not start: %v", work.name, c.Pool, err)) + continue + } else { + said = append(said, fmt.Sprintf("started the %s of %s scheduled %s", work.name, c.Pool, slot.Format("2006-01-02 15:04"))) + } + served[work.key] = slot.Unix() + if err := writeServed(servedPath, served); err != nil { + failed = append(failed, fmt.Errorf("the %s of %s was started, and the served slot could not be kept: %v", work.name, c.Pool, err)) + } + } + if len(said) == 0 { + said = append(said, "nothing due") + } + return strings.Join(said, "; "), joinErrors(failed) +} + +func joinErrors(errs []error) error { + if len(errs) == 0 { + return nil + } + var parts []string + for _, e := range errs { + parts = append(parts, e.Error()) + } + return fmt.Errorf("%s", strings.Join(parts, "; ")) +} diff --git a/modules/zfs/cmd/zfs-tools/run_test.go b/modules/zfs/cmd/zfs-tools/run_test.go new file mode 100644 index 000000000..2be13467b --- /dev/null +++ b/modules/zfs/cmd/zfs-tools/run_test.go @@ -0,0 +1,269 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func scrubConf(pool, cadence, day, at string) string { + return `{"pool":"` + pool + `","scrub-cadence":"` + cadence + `","scrub-day":"` + day + `","scrub-time":"` + at + `"}` +} + +func TestTheSlotIsTheLastScheduledTimeNotAfterNow(t *testing.T) { + loc := time.Local + at := func(y int, m time.Month, d, h, min int) time.Time { return time.Date(y, m, d, h, min, 0, 0, loc) } + cases := []struct { + conf string + now time.Time + want time.Time + }{ + {scrubConf("tank", "weekly", "Mon", "00:00"), at(2026, 10, 9, 12, 0), at(2026, 10, 5, 0, 0)}, + {scrubConf("tank", "weekly", "Fri", "13:30"), at(2026, 10, 9, 12, 0), at(2026, 10, 2, 13, 30)}, + {scrubConf("tank", "weekly", "Fri", "11:30"), at(2026, 10, 9, 12, 0), at(2026, 10, 9, 11, 30)}, + {scrubConf("tank", "monthly", "Mon", "00:00"), at(2026, 10, 9, 12, 0), at(2026, 10, 5, 0, 0)}, + {scrubConf("tank", "monthly", "Mon", "00:00"), at(2026, 10, 4, 23, 0), at(2026, 9, 7, 0, 0)}, + {scrubConf("tank", "monthly", "Thu", "06:00"), at(2026, 10, 1, 6, 0), at(2026, 10, 1, 6, 0)}, + } + for _, c := range cases { + cfg, ref := ParseConfig([]byte(c.conf), 16000) + if len(ref) != 0 { + t.Fatal(ref) + } + s := ScheduleOf(cfg) + if got := s.Slot(c.now); !got.Equal(c.want) { + t.Errorf("%q at %s: slot %s, want %s", c.conf, c.now, got, c.want) + } + if next := s.NextSlot(c.now); !next.After(c.now) || !s.Slot(next).Equal(next) { + t.Errorf("%q at %s: next slot %s", c.conf, c.now, next) + } + } + // A schedule with a value unset is no schedule; a value of another form is refused. + if cfg, _ := ParseConfig([]byte(`{"pool":"tank","scrub-cadence":"weekly"}`), 16000); ScheduleOf(cfg) != nil { + t.Error("a schedule without day and time") + } + for _, conf := range []string{scrubConf("tank", "daily", "Mon", "00:00"), scrubConf("tank", "weekly", "Monday", "00:00"), + scrubConf("tank", "weekly", "Mon", "25:00"), scrubConf("tank x", "weekly", "Mon", "00:00"), scrubConf("tank", "weekly", "Mon..Fri", "00:00"), + scrubConf("tank", "weekly", "Mon", "*:*")} { + if _, ref := ParseConfig([]byte(conf), 16000); len(ref) == 0 { + t.Errorf("accepted %q", conf) + } + } +} + +// runFake is a root machine on the healthy fixtures, with the config file, the schedule state and the lock. +func runFake(t *testing.T, conf string) (*fake, *Machine, memStates, string) { + f := withZFS(t) + f.uid = 0 + f.files[ConfigFile] = conf + f.files[MemInfoFile] = "MemTotal: 16384000 kB\n" + m := f.machine() + st := memStates{} + m.States = st + return f, m, st, filepath.Join(t.TempDir(), "zfs-scrub", "scrub-served.json") +} + +// §20, §24, §31: the lock held: nothing run; a scrub or resilver running or an unimported pool: nothing run +// and said; a due scrub started and the run ends without waiting; a due trim by trim-day and trim-time; a +// served slot not run again; a hand act's key acted on under the lock and deleted. +func TestTheScheduledRunDoesWhatIsDueAndEndsWithoutWaiting(t *testing.T) { + const trim = `{"pool":"tank","trim-schedule":"weekly","trim-day":"Mon","trim-time":"00:00"}` + cases := []struct { + name, conf string + setup func(f *fake, st memStates) + ran, not []string + event string // event kind and reason + served bool + said string + wantErr bool + }{ + {name: "scrub due", conf: scrubConf("tank", "monthly", "Mon", "00:00"), setup: func(f *fake, _ memStates) { f.on("zpool scrub -- tank", "") }, + ran: []string{"zpool scrub -- tank"}, not: []string{"zpool wait"}, served: true, said: "started the scrub"}, + {name: "scrub refused as one just began", conf: scrubConf("tank", "monthly", "Mon", "00:00"), setup: func(f *fake, _ memStates) { + f.answers["zpool scrub -- tank"] = Ran{Code: 1, Err: "cannot scrub tank: currently scrubbing"} + }, ran: []string{"zpool scrub -- tank"}, not: []string{"zpool wait", "scrub -p", "scrub -s"}, served: true}, + {name: "scrub running", conf: scrubConf("tank", "monthly", "Mon", "00:00"), setup: func(f *fake, _ memStates) { + f.on(line("zpool", StatusArgv()), fixture(t, "zpool-status-scrubbing.txt")) + }, not: []string{"zpool scrub", "zpool wait"}, event: "run.skipped/" + SkipScanRunning}, + {name: "resilver running", conf: scrubConf("tank", "monthly", "Mon", "00:00"), setup: func(f *fake, _ memStates) { + f.on(line("zpool", StatusArgv()), fixture(t, "zpool-status-degraded.txt")) + }, not: []string{"zpool scrub"}, event: "run.skipped/" + SkipScanRunning}, + {name: "pool not imported", conf: scrubConf("nopool", "monthly", "Mon", "00:00"), not: []string{"zpool scrub"}, + event: "run.skipped/" + SkipPoolNotImported}, + {name: "slot already served by a scrub that started in it", conf: scrubConf("tank", "weekly", "Sun", "00:00"), setup: func(f *fake, _ memStates) { + f.now = time.Date(2026, 9, 15, 12, 0, 0, 0, time.Local) + }, not: []string{"zpool scrub"}, served: true, said: "ran already"}, + {name: "trim due", conf: trim, setup: func(f *fake, _ memStates) { f.on("zpool trim -- tank", "") }, + ran: []string{"zpool trim -- tank"}, served: true, said: "started the trim"}, + {name: "trim none", conf: `{"pool":"tank","trim-schedule":"none","trim-day":"Mon","trim-time":"00:00"}`, not: []string{"zpool trim"}}, + {name: "a hand act's key is acted on and deleted", conf: `{"pool":"tank"}`, setup: func(f *fake, st memStates) { + st.Put(StateSchedule, "apply", map[string]string{"by": "x"}) + }, said: "apply: applied"}, + {name: "a refused config runs nothing", conf: `{"pool":"tank","scrub-cadence":"hourly"}`, not: []string{"zpool"}, said: "refused"}, + {name: "a failed start fails the run", conf: scrubConf("tank", "monthly", "Mon", "00:00"), setup: func(f *fake, _ memStates) { + f.answers["zpool scrub -- tank"] = Ran{Code: 1, Err: "cannot scrub: no such pool"} + }, ran: []string{"zpool scrub -- tank"}, wantErr: true}, + } + for _, c := range cases { + f, m, st, served := runFake(t, c.conf) + if c.setup != nil { + c.setup(f, st) + } + said, err := m.RunScheduled(context.Background(), served) + if (err != nil) != c.wantErr { + t.Errorf("%s: error %v", c.name, err) + } + for _, r := range c.ran { + if len(ran(f, r)) != 1 { + t.Errorf("%s: %q not run once; ran %v", c.name, r, f.calls) + } + } + for _, n := range c.not { + if len(ran(f, n)) != 0 { + t.Errorf("%s: ran %v", c.name, ran(f, n)) + } + } + if c.event != "" { + kind, reason, _ := strings.Cut(c.event, "/") + if len(f.emitted) != 1 || f.emitted[0]["_type"] != kind || f.emitted[0]["reason"] != reason { + t.Errorf("%s: events %v, want %s", c.name, f.emitted, c.event) + } + } + if _, err := os.Stat(served); (err == nil) != c.served { + t.Errorf("%s: served file present %v, want %v", c.name, err == nil, c.served) + } + if !strings.Contains(said, c.said) { + t.Errorf("%s: said %q, want %q", c.name, said, c.said) + } + if _, ok := st["schedule/apply"]; ok { + t.Errorf("%s: the hand act's key stayed", c.name) + } + } + // The lock held by another: nothing run, said. + f, m, _, served := runFake(t, scrubConf("tank", "monthly", "Mon", "00:00")) + m.Lock = func(string, bool) (func(), bool, error) { return nil, false, nil } + if said, err := m.RunScheduled(context.Background(), served); err != nil || !strings.Contains(said, SkipLockHeld) || + len(f.emitted) != 1 || f.emitted[0]["reason"] != SkipLockHeld || len(ran(f, "zpool")) != 0 { + t.Errorf("lock held: %q %v %v %v", said, err, f.emitted, f.calls) + } +} + +// scheduled is the healthy fixtures with the module's scrub schedule for tank, its timer waiting and its +// last run a success. +func scheduled(t *testing.T) *fake { + f := withZFS(t) + f.files[ConfigFile] = scrubConf("tank", "weekly", "Mon", "00:00") + f.files[MemInfoFile] = "MemTotal: 16384000 kB\n" + f.on(line("systemctl", TimerShowArgv(RunTimer)), + "ActiveState=active\nUnitFileState=enabled\nNextElapseUSecRealtime=@1791842400\nLastTriggerUSec=@1791237600\n") + f.on(line("systemctl", ServiceShowArgv(RunService)), + "ActiveState=inactive\nResult=success\nExecMainStartTimestamp=@1791237600\nExecMainExitTimestamp=@1791331200\n") + return f +} + +func TestTheScheduleIsTheConfigAndWhatTheTimerSays(t *testing.T) { + f := scheduled(t) + got, err := f.machine().Scrubs(context.Background(), false) + if err != nil { + t.Fatal(err) + } + a := got.(ScrubAnswer) + s := a.Schedule + if s == nil || s.Pool != "tank" || s.Cadence != "weekly" || s.Timer != "zfs-mesh-run.timer" || s.TimerState != "active" || + s.Enabled != "enabled" || s.Next == nil || !s.Next.Equal(time.Date(2026, 10, 12, 0, 0, 0, 0, time.Local)) || s.LastResult != "success" { + t.Fatalf("schedule: %+v", s) + } + f = withZFS(t) + got, _ = f.machine().Scrubs(context.Background(), false) + if a := got.(ScrubAnswer); a.Schedule != nil || !strings.Contains(strings.Join(a.Said, " "), "no scrub schedule") { + t.Errorf("no config: %+v", a) + } +} + +func TestTheScheduleWarnsOfAnOldScrubAFailedRunAndAStoppedTimer(t *testing.T) { + f := scheduled(t) + h := f.machine().Health(context.Background()) + if !h.Healthy { + t.Fatalf("%+v", h) + } + w := strings.Join(h.Warnings, " ") + // tank's last scrub finished 2026-09-14, 25 days before the fake clock: past weekly plus its margin. + if !strings.Contains(w, "Pool tank was last scrubbed 25 days ago, more than its weekly schedule allows (9 days)") || + !strings.Contains(w, "The scheduled scrub of tank due 2026-10-05 00:00 has not started") { + t.Errorf("warnings: %v", h.Warnings) + } + f.on(line("systemctl", ServiceShowArgv(RunService)), + "ActiveState=failed\nResult=exit-code\nExecMainStartTimestamp=@1791237600\nExecMainExitTimestamp=@1791237601\n") + f.on(line("systemctl", TimerShowArgv(RunTimer)), + "ActiveState=inactive\nUnitFileState=disabled\nNextElapseUSecRealtime=\nLastTriggerUSec=@1791237600\n") + w = strings.Join(f.machine().Health(context.Background()).Warnings, " ") + for _, want := range []string{"The last scheduled scrub of tank failed (exit-code)", "The scrub schedule's timer zfs-mesh-run.timer is inactive and disabled"} { + if !strings.Contains(w, want) { + t.Errorf("lacks %q: %s", want, w) + } + } +} + +func TestTheScrubActsAreFixedCommandsWithWhyReadBackAndRecorded(t *testing.T) { + for act, cmd := range map[string]string{ + "start": "sudo -n zpool scrub -- tank", + "pause": "sudo -n zpool scrub -p -- tank", + "stop": "sudo -n zpool scrub -s -- tank", + } { + f := withZFS(t) + f.on(cmd, "") + a, err := f.machine().ScrubAct(context.Background(), act, "tank", "the operator asked") + if err != nil { + t.Fatalf("%s: %v", act, err) + } + if strings.Join(a.Ran, " ") != cmd || a.After == "" || !a.Done { + t.Errorf("%s: %+v", act, a) + } + if len(f.emitted) != 1 || f.emitted[0]["_type"] != ScrubEvent || f.emitted[0]["act"] != act || f.emitted[0]["why"] != "the operator asked" { + t.Errorf("%s recorded: %+v", act, f.emitted) + } + } + f := withZFS(t) + for _, c := range []struct{ act, pool, why, says string }{ + {"start", "nope", "why", "no pool"}, + {"start", "tank", "", "say why"}, + {"start", "-s", "why", "no pool"}, + {"destroy", "tank", "why", "not an act"}, + } { + if _, err := f.machine().ScrubAct(context.Background(), c.act, c.pool, c.why); err == nil || !strings.Contains(err.Error(), c.says) { + t.Errorf("%+v: %v", c, err) + } + } + if n := len(ran(f, "sudo")); n != 0 { + t.Errorf("a refusal ran %v", f.calls) + } + f.answers["sudo -n zpool scrub -- tank"] = Ran{Code: -1, TimedOut: true} + if _, err := f.machine().ScrubAct(context.Background(), "start", "tank", "why"); err == nil || !strings.Contains(err.Error(), "outcome unknown; check zfs_scrub") { + t.Errorf("timed out: %v", err) + } +} + +// Third review: the scheduled run's state is root's alone — a folder of its own, 0700 — so the operator account, or anything it runs, cannot make the run write elsewhere. +func TestTheServedSlotIsWrittenOnlyIntoAPrivateFolderAndNeverThroughALink(t *testing.T) { + dir := filepath.Join(t.TempDir(), "zfs-scrub") + path := filepath.Join(dir, "scrub-served.json") + if err := writeServed(path, map[string]int64{"tank": 1}); err != nil { + t.Fatal(err) + } + if fi, err := os.Lstat(dir); err != nil || fi.Mode().Perm() != 0o700 || !fi.IsDir() { + t.Errorf("the folder: %v %v", fi.Mode(), err) + } + // A folder open to others: refused. (The refusal of a link is not run here: no test of this module makes a + // symbolic link, the mesh's hard rule.) + open := filepath.Join(t.TempDir(), "open") + os.Mkdir(open, 0o755) + if err := writeServed(filepath.Join(open, "scrub-served.json"), map[string]int64{"tank": 3}); err == nil { + t.Errorf("written into a folder others can read") + } + if filepath.Dir(ServedFile) == filepath.Dir(ErrorsFile) { + t.Errorf("the run's state shares the account's folder") + } +} diff --git a/modules/zfs/cmd/zfs-tools/schedule.go b/modules/zfs/cmd/zfs-tools/schedule.go index c9e6e4eb8..50c5c217d 100644 --- a/modules/zfs/cmd/zfs-tools/schedule.go +++ b/modules/zfs/cmd/zfs-tools/schedule.go @@ -15,21 +15,19 @@ import ( // The scrub schedule the module owns. // -// The mesh writes ScrubConfig from the assignment's settings. The node-engine runs the module's -// scheduled process, `zfs-tools scrub-run`, from its own timer (ScrubTimer) every quarter of an hour, and -// the run decides from the configuration whether a scrub is due: the last scheduled time not after now -// (the slot) not yet served. Due, it starts the pool's scrub, or waits for one already running, as -// upstream's zfs-scrub@.service does; it never pauses or stops one, and nothing stops the unit to pause -// it (the node-engine writes no ExecStop). So no setting value ever reaches a unit file: the day and the -// time are read and checked here, and an invalid one is refused before anything runs. +// The node-engine writes ConfigFile from the config values. The node-engine runs the module's scheduled +// process, `zfs-tools run`, from its own timer (RunTimer) every quarter of an hour, and the run decides from the +// config whether a scrub is due: the last scheduled time not after now (the slot) not yet served. Due, it starts +// the pool's scrub and returns; it never pauses or stops one and never waits for one (run.go). The day and the +// time are read and checked by the parse (config.go), so no setting value ever reaches a unit file. // // The mesh holds the distribution's own scrub timers for the pool stopped and disabled, so one schedule // scrubs it; the node-engine gives them back running and enabled when the module goes (novox/hq ADR 0269). -// ScrubTimer and ScrubService are the node-engine's units for the scheduled run (the process's name). +// RunTimer and RunService are the node-engine's units for the scheduled run (the process's name). const ( - ScrubTimer = "zfs-mesh-scrub.timer" - ScrubService = "zfs-mesh-scrub.service" + RunTimer = "zfs-mesh-run.timer" + RunService = "zfs-mesh-run.service" ) // ServedFile is where the scheduled run keeps the last slot it served, per pool: in a folder of root's @@ -42,9 +40,6 @@ const ServedFile = "/var/lib/zfs-scrub/scrub-served.json" // fires every quarter of an hour. const MissedAfter = time.Hour -// ScrubConfig is the schedule as the mesh wrote it from the settings. -const ScrubConfig = "/etc/zfs-tools/scrub.conf" - // Margins after a cadence before a last scrub is said to be late: a weekly run that waited for a busy // pool, a monthly run on the first given weekday of a month (28 or 35 days apart). var cadenceAllows = map[string]time.Duration{ @@ -81,33 +76,22 @@ var ( clock = regexp.MustCompile(`^([01][0-9]|2[0-3]):[0-5][0-9]$`) ) -// ParseScrubConfig reads the schedule: pool, cadence (weekly or monthly), day (Mon … Sun) and time (HH:MM). -func ParseScrubConfig(text string) (*Schedule, error) { - v := map[string]string{} - for _, l := range strings.Split(text, "\n") { - l = strings.TrimSpace(l) - if l == "" || strings.HasPrefix(l, "#") { - continue - } - k, val, ok := strings.Cut(l, "=") - if !ok { - return nil, fmt.Errorf("%s: %q is not key=value", ScrubConfig, l) - } - v[strings.TrimSpace(k)] = strings.TrimSpace(val) +// ScheduleOf is the scrub schedule the config holds: the pool, and the cadence, day and time. nil when the +// values are not all set (the module then has no scrub schedule, §11: not managed until set). +func ScheduleOf(c *Config) *Schedule { + if c.Pool == "" || c.ScrubCadence == "" || c.ScrubDay == "" || c.ScrubTime == "" { + return nil } - s := &Schedule{Pool: v["pool"], Cadence: v["cadence"], Day: v["day"], Time: v["time"]} - switch { - case !poolName.MatchString(s.Pool): - return nil, fmt.Errorf("the scrub-pool setting %q is not a pool's name", s.Pool) - case cadenceAllows[s.Cadence] == 0: - return nil, fmt.Errorf("the scrub-cadence setting is %q; it is weekly or monthly", s.Cadence) - case !weekday.MatchString(s.Day): - return nil, fmt.Errorf("the scrub-day setting is %q; it is Mon, Tue, Wed, Thu, Fri, Sat or Sun", s.Day) - case !clock.MatchString(s.Time): - return nil, fmt.Errorf("the scrub-time setting is %q; it is HH:MM, 00:00 to 23:59", s.Time) + return &Schedule{Pool: c.Pool, Cadence: c.ScrubCadence, Day: c.ScrubDay, Time: c.ScrubTime, Timer: RunTimer, Service: RunService} +} + +// TrimScheduleOf is the trim schedule: nil while trim-schedule is not weekly or monthly, or its day or time is +// not set. +func TrimScheduleOf(c *Config) *Schedule { + if c.Pool == "" || (c.TrimSchedule != "weekly" && c.TrimSchedule != "monthly") || c.TrimDay == "" || c.TrimTime == "" { + return nil } - s.Timer, s.Service = ScrubTimer, ScrubService - return s, nil + return &Schedule{Pool: c.Pool, Cadence: c.TrimSchedule, Day: c.TrimDay, Time: c.TrimTime, Timer: RunTimer, Service: RunService} } var weekdays = map[string]time.Weekday{"Sun": time.Sunday, "Mon": time.Monday, "Tue": time.Tuesday, @@ -150,13 +134,6 @@ func (s *Schedule) NextSlot(now time.Time) time.Time { return time.Time{} } -// WaitArgv waits for a pool's running scrub to end. -func WaitArgv(pool string) []string { return []string{"wait", "-t", "scrub", "--", pool} } - -// WaitBound is how long the scheduled run waits for a scrub: past it the run ends and leaves the scrub -// running; a scrub never pauses for it. -const WaitBound = 7 * 24 * time.Hour - // readServed is the last slot served per pool; none when the file is not there or not readable. func (m *Machine) readServed(path string) map[string]int64 { out := map[string]int64{} @@ -220,72 +197,6 @@ func startedSince(p *PoolStatus, slot time.Time) bool { return sc != nil && sc.Started != nil && !sc.Started.Before(slot) } -// ScrubRun is the scheduled run: a due scrub started, or one running waited for. It answers what it did in -// a line, and fails on an invalid schedule, which it never acts on. -func (m *Machine) ScrubRun(ctx context.Context, servedPath string) (string, error) { - raw, err := m.ReadFile(ScrubConfig) - if err != nil { - return "", fmt.Errorf("%s could not be read: %v", ScrubConfig, err) - } - s, err := ParseScrubConfig(string(raw)) - if err != nil { - return "", fmt.Errorf("refused, nothing run: %v", err) - } - now := m.Now() - slot := s.Slot(now) - served := m.readServed(servedPath) - if served[s.Pool] >= slot.Unix() { - return fmt.Sprintf("the scrub of %s scheduled %s was served", s.Pool, slot.Format("2006-01-02 15:04")), nil - } - st, err := m.Status(ctx) - if err != nil { - return "", err - } - p := statusOf(st, s.Pool) - if p == nil { - return "", fmt.Errorf("the scheduled pool %s is not imported here", s.Pool) - } - mark := func() error { - served[s.Pool] = slot.Unix() - return writeServed(servedPath, served) - } - if startedSince(p, slot) && p.Scrub().State != "running" { - return fmt.Sprintf("the scrub of %s scheduled %s ran already", s.Pool, slot.Format("2006-01-02 15:04")), mark() - } - for _, r := range p.Resilvers() { - if r.State == "running" { - return fmt.Sprintf("a resilver of %s runs; the scrub scheduled %s waits for the next run after it", s.Pool, - slot.Format("2006-01-02 15:04")), nil - } - } - started := false - if sc := p.Scrub(); sc == nil || sc.State != "running" { - _, err := m.AsRoot(ctx, "zpool", ScrubArgv("start", s.Pool)...) - switch { - case err == nil: - started = true - case strings.Contains(err.Error(), "currently scrubbing"): - // One began since status was read: it serves the slot, and is waited for like any other. - default: - return "", fmt.Errorf("the scheduled scrub of %s could not start: %v", s.Pool, err) - } - } - if err := mark(); err != nil { - return "", fmt.Errorf("the scrub of %s runs, and the served slot could not be kept: %v", s.Pool, err) - } - wctx, cancel := context.WithTimeout(ctx, WaitBound) - defer cancel() - _, werr := m.AsRoot(wctx, "zpool", WaitArgv(s.Pool)...) - what := "waited for the scrub of " + s.Pool + " already running" - if started { - what = "started the scrub of " + s.Pool + " scheduled " + slot.Format("2006-01-02 15:04") + ", and waited for it" - } - if werr != nil { - return what + "; the wait ended without its end (" + werr.Error() + "), and the scrub goes on", nil - } - return what + " to finish", nil -} - // TimerShowArgv asks the service manager what a timer says: whether it runs and is enabled, when it // fires next and when it last did, as Unix times. func TimerShowArgv(unit string) []string { @@ -320,15 +231,19 @@ func unixStamp(s string) *time.Time { } // ScheduleNow is the schedule with what the service manager says of it; nil when the module has no -// schedule on this machine (no configuration written). +// schedule on this machine (no config file, or no cadence, day and time set). func (m *Machine) ScheduleNow(ctx context.Context) (*Schedule, error) { - raw, err := m.ReadFile(ScrubConfig) + raw, err := m.ReadFile(ConfigFile) if err != nil { return nil, nil } - s, err := ParseScrubConfig(string(raw)) - if err != nil { - return nil, err + c, refused := ParseConfig(raw, m.memMiB()) + if len(refused) > 0 { + return nil, fmt.Errorf("%s refused %s %s", ConfigFile, refused[0].Key, refused[0].Reason) + } + s := ScheduleOf(c) + if s == nil { + return nil, nil } if out, err := m.ask(ctx, "systemctl", TimerShowArgv(s.Timer)...); err != nil { s.Note = "the service manager did not say what the timer is: " + err.Error() diff --git a/modules/zfs/cmd/zfs-tools/scrub.go b/modules/zfs/cmd/zfs-tools/scrub.go index 85e9a3b4e..aa79acff7 100644 --- a/modules/zfs/cmd/zfs-tools/scrub.go +++ b/modules/zfs/cmd/zfs-tools/scrub.go @@ -141,7 +141,7 @@ func (m *Machine) Scrubs(ctx context.Context, history bool) (any, error) { case serr != nil: a.Said = append(a.Said, "The scrub schedule's settings are not valid: "+serr.Error()+".") case sched == nil: - a.Said = append(a.Said, "The module has no scrub schedule on this machine (no "+ScrubConfig+").") + a.Said = append(a.Said, "The module has no scrub schedule on this machine (no cadence, day and time set in "+ConfigFile+").") default: a.Schedule = sched line := fmt.Sprintf("%s scrub of %s, %s at %s (monthly: the first %s of the month), decided by the run %s fires "+ diff --git a/modules/zfs/cmd/zfs-tools/scrubrun_test.go b/modules/zfs/cmd/zfs-tools/scrubrun_test.go deleted file mode 100644 index 857256e07..000000000 --- a/modules/zfs/cmd/zfs-tools/scrubrun_test.go +++ /dev/null @@ -1,206 +0,0 @@ -package main - -import ( - "context" - "os" - "path/filepath" - "strings" - "testing" - "time" -) - -// The scheduled run (`zfs-tools scrub-run`, second review of mesh-catalog #147): the node-engine's timer -// fires it every quarter of an hour, and it decides from the configuration whether a scrub is due. It -// starts one, or waits for one already running, as upstream's zfs-scrub@.service does, and never pauses or -// stops a scrub. - -func monthlyConf(pool string) string { - return "pool=" + pool + "\ncadence=monthly\nday=Mon\ntime=00:00\n" -} - -func TestTheSlotIsTheLastScheduledTimeNotAfterNow(t *testing.T) { - loc := time.Local - at := func(y int, m time.Month, d, h, min int) time.Time { return time.Date(y, m, d, h, min, 0, 0, loc) } - cases := []struct { - conf string - now time.Time - want time.Time - }{ - {"pool=tank\ncadence=weekly\nday=Mon\ntime=00:00\n", at(2026, 10, 9, 12, 0), at(2026, 10, 5, 0, 0)}, - {"pool=tank\ncadence=weekly\nday=Fri\ntime=13:30\n", at(2026, 10, 9, 12, 0), at(2026, 10, 2, 13, 30)}, - {"pool=tank\ncadence=weekly\nday=Fri\ntime=11:30\n", at(2026, 10, 9, 12, 0), at(2026, 10, 9, 11, 30)}, - {monthlyConf("tank"), at(2026, 10, 9, 12, 0), at(2026, 10, 5, 0, 0)}, - {monthlyConf("tank"), at(2026, 10, 4, 23, 0), at(2026, 9, 7, 0, 0)}, - {"pool=tank\ncadence=monthly\nday=Thu\ntime=06:00\n", at(2026, 10, 1, 6, 0), at(2026, 10, 1, 6, 0)}, - } - for _, c := range cases { - s, err := ParseScrubConfig(c.conf) - if err != nil { - t.Fatal(err) - } - if got := s.Slot(c.now); !got.Equal(c.want) { - t.Errorf("%q at %s: slot %s, want %s", c.conf, c.now, got, c.want) - } - if next := s.NextSlot(c.now); !next.After(c.now) || !s.Slot(next).Equal(next) { - t.Errorf("%q at %s: next slot %s", c.conf, c.now, next) - } - } -} - -// root runs the scheduled run: no sudo. -func rootFake(t *testing.T, conf string) (*fake, string) { - f := withZFS(t) - f.uid = 0 - f.files[ScrubConfig] = conf - dir := filepath.Join(t.TempDir(), "zfs-scrub") - return f, filepath.Join(dir, "scrub-served.json") -} - -func TestADueScrubIsStartedThenWaitedForAndMarkedServed(t *testing.T) { - f, served := rootFake(t, monthlyConf("tank")) - f.on("zpool scrub -- tank", "") - f.on("zpool wait -t scrub -- tank", "") - said, err := f.machine().ScrubRun(context.Background(), served) - if err != nil { - t.Fatal(err) - } - if got := strings.Join(ran(f, "zpool scrub"), "|") + "/" + strings.Join(ran(f, "zpool wait"), "|"); got != "zpool scrub -- tank/zpool wait -t scrub -- tank" { - t.Errorf("ran %q", got) - } - if !strings.Contains(said, "started") { - t.Errorf("said %q", said) - } - // Served: the next fire within the same slot runs nothing. - f.calls = nil - if _, err := f.machine().ScrubRun(context.Background(), served); err != nil { - t.Fatal(err) - } - if n := len(ran(f, "zpool scrub")) + len(ran(f, "zpool wait")); n != 0 { - t.Errorf("a served slot ran %v", f.calls) - } -} - -func TestAScrubInProgressIsWaitedForNeverPausedOrStarted(t *testing.T) { - f, served := rootFake(t, monthlyConf("tank")) - f.on(line("zpool", StatusArgv()), fixture(t, "zpool-status-scrubbing.txt")) - f.on(line("zpool", PoolsArgv()), "tank\t1000\t100\t900\t10\t1\tONLINE\t1.00\n") - f.on("zpool wait -t scrub -- tank", "") - if _, err := f.machine().ScrubRun(context.Background(), served); err != nil { - t.Fatal(err) - } - if strings.Join(f.calls, "|") == "" || len(ran(f, "zpool scrub")) != 0 || len(ran(f, "zpool wait -t scrub -- tank")) != 1 { - t.Errorf("ran %v", f.calls) - } -} - -func TestAStartRefusedBecauseAScrubJustBeganIsWaitedForNotPaused(t *testing.T) { - f, served := rootFake(t, monthlyConf("tank")) - f.answers["zpool scrub -- tank"] = Ran{Code: 1, Err: "cannot scrub tank: currently scrubbing; use 'zpool scrub -s' to cancel the current scrub"} - f.on("zpool wait -t scrub -- tank", "") - if _, err := f.machine().ScrubRun(context.Background(), served); err != nil { - t.Fatal(err) - } - for _, c := range f.calls { - if strings.Contains(c, "scrub -p") || strings.Contains(c, "scrub -s") { - t.Errorf("the run paused or stopped a scrub: %s", c) - } - } - if len(ran(f, "zpool wait -t scrub -- tank")) != 1 { - t.Errorf("not waited for: %v", f.calls) - } -} - -func TestAResilverDefersTheScrubWithoutServingTheSlot(t *testing.T) { - f, served := rootFake(t, monthlyConf("tank")) - f.on(line("zpool", StatusArgv()), fixture(t, "zpool-status-degraded.txt")) - f.on(line("zpool", PoolsArgv()), "tank\t1000\t100\t900\t10\t1\tDEGRADED\t1.00\n") - said, err := f.machine().ScrubRun(context.Background(), served) - if err != nil { - t.Fatal(err) - } - if len(ran(f, "zpool scrub")) != 0 || !strings.Contains(said, "resilver") { - t.Errorf("ran %v, said %q", f.calls, said) - } - if _, err := os.Stat(served); err == nil { - t.Errorf("the slot was marked served while the scrub waits for the resilver") - } -} - -func TestAScrubAlreadyStartedInTheSlotServesItWithoutAStateFile(t *testing.T) { - // tank's scrub started 2026-09-13 00:24 (finished 09-14); a weekly Sunday-00:00 slot of 09-13 is served - // by it, so a run on 09-15 starts nothing. - f, served := rootFake(t, "pool=tank\ncadence=weekly\nday=Sun\ntime=00:00\n") - f.now = time.Date(2026, 9, 15, 12, 0, 0, 0, time.Local) - if _, err := f.machine().ScrubRun(context.Background(), served); err != nil { - t.Fatal(err) - } - if n := len(ran(f, "zpool scrub")) + len(ran(f, "zpool wait")); n != 0 { - t.Errorf("ran %v", f.calls) - } -} - -func TestAnInvalidScheduleRunsNothingAndFails(t *testing.T) { - for _, conf := range []string{ - "pool=tank\ncadence=monthly\nday=Mon\ntime=*:*\n", - "pool=tank\ncadence=monthly\nday=Mon..Fri\ntime=00:00\n", - "pool=tank\ncadence=monthly\nday=*\ntime=00:00\n", - "pool=tank\ncadence=hourly\nday=Mon\ntime=00:00\n", - "pool=tank\ncadence=monthly\nday=Mon\ntime=0:00\n", - } { - f, served := rootFake(t, conf) - if _, err := f.machine().ScrubRun(context.Background(), served); err == nil { - t.Errorf("%q: not refused", conf) - } - if len(ran(f, "zpool scrub"))+len(ran(f, "zpool wait")) != 0 { - t.Errorf("%q ran %v", conf, f.calls) - } - } -} - -func TestAScheduledScrubThatDidNotStartIsAWarning(t *testing.T) { - f := withZFS(t) - f.files[ScrubConfig] = "pool=tank\ncadence=weekly\nday=Mon\ntime=00:00\n" - f.on(line("systemctl", TimerShowArgv(ScrubTimer)), "ActiveState=active\nUnitFileState=enabled\nNextElapseUSecRealtime=@1791842400\nLastTriggerUSec=@1791237600\n") - h := f.machine().Health(context.Background()) - if !strings.Contains(strings.Join(h.Warnings, " "), "The scheduled scrub of tank due 2026-10-05 00:00 has not started") { - t.Errorf("warnings: %v", h.Warnings) - } -} - -// Third review: the scheduled run's state is root's alone — a folder of its own, 0700, never through a -// link — so the operator account, or anything it runs, cannot make the run write elsewhere. -func TestTheServedSlotIsWrittenOnlyIntoAPrivateFolderAndNeverThroughALink(t *testing.T) { - dir := filepath.Join(t.TempDir(), "zfs-scrub") - path := filepath.Join(dir, "scrub-served.json") - if err := writeServed(path, map[string]int64{"tank": 1}); err != nil { - t.Fatal(err) - } - if fi, err := os.Lstat(dir); err != nil || fi.Mode().Perm() != 0o700 || !fi.IsDir() { - t.Errorf("the folder: %v %v", fi.Mode(), err) - } - // The file replaced by a link to somewhere else: refused, and the other place untouched. - victim := filepath.Join(t.TempDir(), "victim") - os.WriteFile(victim, []byte("keep"), 0o644) - os.Remove(path) - os.Symlink(victim, path) - if err := writeServed(path, map[string]int64{"tank": 2}); err == nil { - t.Errorf("written through a link") - } - if b, _ := os.ReadFile(victim); string(b) != "keep" { - t.Errorf("the link's target was changed: %q", b) - } - // The folder itself a link, or open to others: refused. - linked := filepath.Join(t.TempDir(), "linked") - os.Symlink(t.TempDir(), linked) - if err := writeServed(filepath.Join(linked, "scrub-served.json"), map[string]int64{"tank": 3}); err == nil { - t.Errorf("written into a linked folder") - } - open := filepath.Join(t.TempDir(), "open") - os.Mkdir(open, 0o755) - if err := writeServed(filepath.Join(open, "scrub-served.json"), map[string]int64{"tank": 3}); err == nil { - t.Errorf("written into a folder others can read") - } - if filepath.Dir(ServedFile) == filepath.Dir(ErrorsFile) { - t.Errorf("the run's state shares the account's folder") - } -} diff --git a/modules/zfs/cmd/zfs-tools/scrubschedule_test.go b/modules/zfs/cmd/zfs-tools/scrubschedule_test.go deleted file mode 100644 index eaa8acf6d..000000000 --- a/modules/zfs/cmd/zfs-tools/scrubschedule_test.go +++ /dev/null @@ -1,119 +0,0 @@ -package main - -import ( - "context" - "strings" - "testing" - "time" -) - -const scrubConf = "pool=tank\ncadence=weekly\nday=Mon\ntime=00:00\n" - -// scheduled is the healthy fixtures with the module's scrub schedule for tank, its timer waiting and its -// last run a success. -func scheduled(t *testing.T) *fake { - f := withZFS(t) - f.files[ScrubConfig] = scrubConf - f.on(line("systemctl", TimerShowArgv(ScrubTimer)), - "ActiveState=active\nUnitFileState=enabled\nNextElapseUSecRealtime=@1791842400\nLastTriggerUSec=@1791237600\n") - f.on(line("systemctl", ServiceShowArgv(ScrubService)), - "ActiveState=inactive\nResult=success\nExecMainStartTimestamp=@1791237600\nExecMainExitTimestamp=@1791331200\n") - return f -} - -func TestTheScheduleIsTheConfigAndWhatTheTimerSays(t *testing.T) { - f := scheduled(t) - got, err := f.machine().Scrubs(context.Background(), false) - if err != nil { - t.Fatal(err) - } - a := got.(ScrubAnswer) - s := a.Schedule - if s == nil || s.Pool != "tank" || s.Cadence != "weekly" || s.Timer != "zfs-mesh-scrub.timer" || s.TimerState != "active" || - s.Enabled != "enabled" || s.Next == nil || !s.Next.Equal(time.Date(2026, 10, 12, 0, 0, 0, 0, time.Local)) || s.LastResult != "success" { - t.Fatalf("schedule: %+v", s) - } - if want := s.NextSlot(f.now).Format("2006-01-02 15:04"); !strings.Contains(strings.Join(a.Said, "\n"), "next scheduled scrub of tank: "+want) || want != "2026-10-12 00:00" { - t.Errorf("said: %v", a.Said) - } -} - -func TestAScheduleThatIsNotSetIsSaidAndAnInvalidOneRefused(t *testing.T) { - f := withZFS(t) - got, _ := f.machine().Scrubs(context.Background(), false) - if a := got.(ScrubAnswer); a.Schedule != nil || !strings.Contains(strings.Join(a.Said, " "), "no scrub schedule") { - t.Errorf("no config: %+v", a) - } - for _, conf := range []string{ - "pool=tank\ncadence=daily\nday=Mon\ntime=00:00\n", - "pool=tank\ncadence=weekly\nday=Monday\ntime=00:00\n", - "pool=tank\ncadence=weekly\nday=Mon\ntime=25:00\n", - "pool=tank x\ncadence=weekly\nday=Mon\ntime=00:00\n", - } { - if _, err := ParseScrubConfig(conf); err == nil { - t.Errorf("accepted %q", conf) - } - } -} - -func TestTheScheduleWarnsOfAnOldScrubAFailedRunAndAStoppedTimer(t *testing.T) { - f := scheduled(t) - h := f.machine().Health(context.Background()) - if !h.Healthy { - t.Fatalf("%+v", h) - } - w := strings.Join(h.Warnings, " ") - // tank's last scrub finished 2026-09-14, 25 days before the fake clock: past weekly plus its margin. - if !strings.Contains(w, "Pool tank was last scrubbed 25 days ago, more than its weekly schedule allows (9 days)") { - t.Errorf("warnings: %v", h.Warnings) - } - f.on(line("systemctl", ServiceShowArgv(ScrubService)), - "ActiveState=failed\nResult=exit-code\nExecMainStartTimestamp=@1791237600\nExecMainExitTimestamp=@1791237601\n") - f.on(line("systemctl", TimerShowArgv(ScrubTimer)), - "ActiveState=inactive\nUnitFileState=disabled\nNextElapseUSecRealtime=\nLastTriggerUSec=@1791237600\n") - w = strings.Join(f.machine().Health(context.Background()).Warnings, " ") - for _, want := range []string{"The last scheduled scrub of tank failed (exit-code)", "The scrub schedule's timer zfs-mesh-scrub.timer is inactive and disabled"} { - if !strings.Contains(w, want) { - t.Errorf("lacks %q: %s", want, w) - } - } -} - -func TestTheScrubActsAreFixedCommandsWithWhyReadBackAndRecorded(t *testing.T) { - for act, cmd := range map[string]string{ - "start": "sudo -n zpool scrub -- tank", - "pause": "sudo -n zpool scrub -p -- tank", - "stop": "sudo -n zpool scrub -s -- tank", - } { - f := withZFS(t) - f.on(cmd, "") - a, err := f.machine().ScrubAct(context.Background(), act, "tank", "the operator asked") - if err != nil { - t.Fatalf("%s: %v", act, err) - } - if strings.Join(a.Ran, " ") != cmd || a.After == "" || !a.Done { - t.Errorf("%s: %+v", act, a) - } - if len(f.emitted) != 1 || f.emitted[0]["_type"] != ScrubEvent || f.emitted[0]["act"] != act || f.emitted[0]["why"] != "the operator asked" { - t.Errorf("%s recorded: %+v", act, f.emitted) - } - } - f := withZFS(t) - for _, c := range []struct{ act, pool, why, says string }{ - {"start", "nope", "why", "no pool"}, - {"start", "tank", "", "say why"}, - {"start", "-s", "why", "no pool"}, - {"destroy", "tank", "why", "not an act"}, - } { - if _, err := f.machine().ScrubAct(context.Background(), c.act, c.pool, c.why); err == nil || !strings.Contains(err.Error(), c.says) { - t.Errorf("%+v: %v", c, err) - } - } - if n := len(ran(f, "sudo")); n != 0 { - t.Errorf("a refusal ran %v", f.calls) - } - f.answers["sudo -n zpool scrub -- tank"] = Ran{Code: -1, TimedOut: true} - if _, err := f.machine().ScrubAct(context.Background(), "start", "tank", "why"); err == nil || !strings.Contains(err.Error(), "outcome unknown; check zfs_scrub") { - t.Errorf("timed out: %v", err) - } -} diff --git a/modules/zfs/module.json b/modules/zfs/module.json index 195263374..bf871502e 100644 --- a/modules/zfs/module.json +++ b/modules/zfs/module.json @@ -187,13 +187,13 @@ "unknown": "program" }, { - "id": "scrub-run", + "id": "run", "type": "process", - "name": "zfs-mesh-scrub", + "name": "zfs-mesh-run", "artifact": "tools", "run": [ "./zfs-tools", - "scrub-run" + "run" ], "schedule": "0,15,30,45 * * * *" },