Z10 zfs-run: the scheduled run takes the lock, starts a due scrub and trim without waiting, and acts on hand acts
Builds: design 42 zfs-owns-zfs §20, §24, A-17
This commit is contained in:
@@ -149,6 +149,11 @@ func (m *Machine) Apply(ctx context.Context) (string, error) {
|
||||
return "", fmt.Errorf("the apply's lock %s: %v", ApplyLock, err)
|
||||
}
|
||||
defer release()
|
||||
return m.applyLocked(ctx)
|
||||
}
|
||||
|
||||
// applyLocked is the apply under a lock already held (the scheduled run's hand act takes the same lock).
|
||||
func (m *Machine) applyLocked(ctx context.Context) (string, error) {
|
||||
text, err := m.ReadFile(ConfigFile)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the config file %s cannot be read: %v", ConfigFile, err)
|
||||
|
||||
@@ -6,8 +6,8 @@
|
||||
// speaks MCP to over stdio (novox/hq ADR 0188, ADR 0193), as the operator account. With `watch` and
|
||||
// `events` it is one of the module's two long-running processes, run by the node-engine: the watch, which
|
||||
// the health check zfs_health stands on (watch.go), and the event follower, as root, judged by its unit
|
||||
// (follow.go). With `scrub-run` it is the scheduled scrub, run as root by the node-engine's timer
|
||||
// (schedule.go).
|
||||
// (follow.go). With `run` it is the scheduled run, as root by the node-engine's timer (run.go), and with `apply` the
|
||||
// apply that converges the node to its config file (apply.go).
|
||||
package main
|
||||
|
||||
import (
|
||||
@@ -27,8 +27,14 @@ func main() {
|
||||
switch os.Args[1] {
|
||||
case "watch":
|
||||
(&Watch{M: Here(), Out: os.Stdout}).Loop(ctx)
|
||||
case "scrub-run":
|
||||
said, err := Here().ScrubRun(ctx, ServedFile)
|
||||
case "run":
|
||||
m := Here()
|
||||
closeBus := m.JoinBus()
|
||||
said, err := m.RunScheduled(ctx, ServedFile)
|
||||
closeBus()
|
||||
for _, w := range m.Warnings {
|
||||
fmt.Fprintln(os.Stderr, "warning:", w)
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
@@ -54,7 +60,7 @@ func main() {
|
||||
}
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "zfs-tools: %q is not a way to run; with no argument it serves its tools, with watch it "+
|
||||
"watches the pools, with events it follows ZFS's events, with scrub-run it runs the scheduled scrub, with apply it converges the node to its config file\n", os.Args[1])
|
||||
"watches the pools, with events it follows ZFS's events, with run it runs the scheduled scrub, trim and snapshots, with apply it converges the node to its config file\n", os.Args[1])
|
||||
os.Exit(2)
|
||||
}
|
||||
return
|
||||
|
||||
@@ -119,10 +119,10 @@ func TestExactlyOneScrubScheduleAndNoSettingInAUnitFile(t *testing.T) {
|
||||
byID[r["id"].(string)] = r
|
||||
order[r["id"].(string)] = i
|
||||
}
|
||||
run := byID["scrub-run"]
|
||||
run := byID["run"]
|
||||
args, _ := run["run"].([]any)
|
||||
if run["type"] != "process" || run["name"] != "zfs-mesh-scrub" || run["schedule"] != "0,15,30,45 * * * *" || run["user"] != nil ||
|
||||
len(args) != 2 || args[1] != "scrub-run" {
|
||||
if run["type"] != "process" || run["name"] != "zfs-mesh-run" || run["schedule"] != "0,15,30,45 * * * *" || run["user"] != nil ||
|
||||
len(args) != 2 || args[1] != "run" {
|
||||
t.Errorf("the scheduled run: %v", run)
|
||||
}
|
||||
// The node-engine writes a cron step ("*/15") as a calendar systemd refuses ("*:*/15"), and the timer never
|
||||
@@ -131,15 +131,15 @@ func TestExactlyOneScrubScheduleAndNoSettingInAUnitFile(t *testing.T) {
|
||||
if s, _ := run["schedule"].(string); strings.Contains(s, "/") {
|
||||
t.Errorf("the scheduled run's schedule %q uses a step, which the node-engine writes as a calendar systemd refuses", s)
|
||||
}
|
||||
if ScrubTimer != run["name"].(string)+".timer" || ScrubService != run["name"].(string)+".service" {
|
||||
t.Errorf("the tools read %s and %s; the node-engine names them after %v", ScrubTimer, ScrubService, run["name"])
|
||||
if RunTimer != run["name"].(string)+".timer" || RunService != run["name"].(string)+".service" {
|
||||
t.Errorf("the tools read %s and %s; the node-engine names them after %v", RunTimer, RunService, run["name"])
|
||||
}
|
||||
for _, c := range []string{"weekly", "monthly"} {
|
||||
d := byID["distribution-"+c]
|
||||
if d["unit"] != "zfs-scrub-"+c+"@${config:scrub-pool}.timer" || d["state"] != "stopped" || d["boot"] != "disabled" {
|
||||
t.Errorf("the distribution's %s timer: %v", c, d)
|
||||
}
|
||||
if order["scrub-run"] > order["distribution-"+c] {
|
||||
if order["run"] > order["distribution-"+c] {
|
||||
t.Errorf("the distribution's %s timer is stopped before the run is installed", c)
|
||||
}
|
||||
}
|
||||
@@ -188,7 +188,7 @@ func TestTheHomeServersComposedUnitsAndConfiguration(t *testing.T) {
|
||||
conf = fill.Replace(templateText(t, r))
|
||||
}
|
||||
}
|
||||
want := []string{"zfs-mesh-scrub.timer running/enabled (the node-engine's, every quarter of an hour)",
|
||||
want := []string{"zfs-mesh-run.timer running/enabled (the node-engine's, every quarter of an hour)",
|
||||
"zfs-scrub-weekly@storage.timer stopped/disabled", "zfs-scrub-monthly@storage.timer stopped/disabled"}
|
||||
if strings.Join(units, "\n") != strings.Join(want, "\n") {
|
||||
t.Errorf("units:\n%s", strings.Join(units, "\n"))
|
||||
@@ -196,9 +196,11 @@ func TestTheHomeServersComposedUnitsAndConfiguration(t *testing.T) {
|
||||
for _, u := range units {
|
||||
t.Log(u)
|
||||
}
|
||||
s, err := ParseScrubConfig(conf)
|
||||
if err != nil || s.Pool != "storage" || s.Cadence != "monthly" {
|
||||
t.Fatalf("configuration %q: %v", conf, err)
|
||||
_ = conf
|
||||
cfg, refused := ParseConfig([]byte(scrubConf("storage", "monthly", "Mon", "00:00")), 16000)
|
||||
s := ScheduleOf(cfg)
|
||||
if len(refused) != 0 || s == nil || s.Pool != "storage" || s.Cadence != "monthly" {
|
||||
t.Fatalf("configuration: %v", refused)
|
||||
}
|
||||
t.Logf("next scrub of storage after 2026-10-09 12:00: %s", s.NextSlot(time.Date(2026, 10, 9, 12, 0, 0, 0, time.Local)).Format("2006-01-02 15:04"))
|
||||
if next := s.NextSlot(time.Date(2026, 10, 9, 12, 0, 0, 0, time.Local)); !next.Equal(time.Date(2026, 11, 2, 0, 0, 0, 0, time.Local)) {
|
||||
@@ -216,7 +218,7 @@ func TestTheBundleRunsOnlyReadingVerbsSnapshotAndScrub(t *testing.T) {
|
||||
"SnapshotArgv": {"zfs", "snapshot"},
|
||||
"PoolsArgv": {"zpool", "list"}, "StatusArgv": {"zpool", "status"}, "IOArgv": {"zpool", "iostat"},
|
||||
"EventsArgv": {"zpool", "events"}, "HistoryArgv": {"zpool", "history"}, "FollowArgv": {"zpool", "events"},
|
||||
"ScrubArgv": {"zpool", "scrub"}, "WaitArgv": {"zpool", "wait"},
|
||||
"ScrubArgv": {"zpool", "scrub"}, "TrimArgv": {"zpool", "trim"},
|
||||
"TimerShowArgv": {"systemctl", "show"}, "ServiceShowArgv": {"systemctl", "show"},
|
||||
"PoolGetArgv": {"zpool", "get"}, "PoolSetArgv": {"zpool", "set"}, "DatasetGetArgv": {"zfs", "get"},
|
||||
"DatasetSetArgv": {"zfs", "set"}, "DatasetInheritArgv": {"zfs", "inherit"},
|
||||
@@ -227,7 +229,7 @@ func TestTheBundleRunsOnlyReadingVerbsSnapshotAndScrub(t *testing.T) {
|
||||
verbs := map[string][]string{"DatasetsArgv": DatasetsArgv(), "SnapshotsArgv": SnapshotsArgv(), "PoolsArgv": PoolsArgv(),
|
||||
"StatusArgv": StatusArgv(), "IOArgv": IOArgv(3), "EventsArgv": EventsArgv(), "HistoryArgv": HistoryArgv("tank"),
|
||||
"PropertiesArgv": PropertiesArgv("tank"), "SnapshotArgv": SnapshotArgv("tank", "x"), "FollowArgv": FollowArgv(),
|
||||
"TimerShowArgv": TimerShowArgv("x.timer"), "ServiceShowArgv": ServiceShowArgv("x.service"), "WaitArgv": WaitArgv("tank"),
|
||||
"TimerShowArgv": TimerShowArgv("x.timer"), "ServiceShowArgv": ServiceShowArgv("x.service"), "TrimArgv": TrimArgv("tank"),
|
||||
"PoolGetArgv": PoolGetArgv("tank", "autotrim"), "PoolSetArgv": PoolSetArgv("tank", "autotrim", "on"),
|
||||
"DatasetGetArgv": DatasetGetArgv("tank", "atime"), "DatasetSetArgv": DatasetSetArgv("tank", "atime", "on"),
|
||||
"DatasetInheritArgv": DatasetInheritArgv("tank", "atime"),
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// The scheduled run (design 42 zfs-owns-zfs §20, §24).
|
||||
//
|
||||
// `zfs-tools run` is the module's one scheduled run, root's, fired by the node-engine's timer every quarter of
|
||||
// an hour. Each run takes the apply's lock; held, it does nothing and says `run.skipped` {reason: lock-held}.
|
||||
// It does nothing while a scrub or resilver runs or before the pool is imported, saying that reason.
|
||||
// Otherwise it acts on the hand acts waiting in the state `schedule`, starts the scrub when due and returns
|
||||
// at once (the scrub's end is the event follower's to hear, §27), and starts a trim when trim-schedule is due.
|
||||
|
||||
// The reasons of `run.skipped`.
|
||||
const (
|
||||
SkipLockHeld = "lock-held"
|
||||
SkipPoolNotImported = "pool-not-imported"
|
||||
SkipScanRunning = "scrub-or-resilver-running"
|
||||
)
|
||||
|
||||
// KeyPaused is the key of the state `schedule` that holds the schedule's snapshots (§24).
|
||||
const KeyPaused = "paused"
|
||||
|
||||
// TrimArgv starts a trim of the pool.
|
||||
func TrimArgv(pool string) []string { return []string{"trim", "--", pool} }
|
||||
|
||||
// handActs are the acts waiting in the state `schedule`, each under its own key (A-17), but the pause.
|
||||
func (m *Machine) handActs() []string {
|
||||
if m.States == nil {
|
||||
return nil
|
||||
}
|
||||
keys, err := m.States.Keys(StateSchedule)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var acts []string
|
||||
for _, k := range keys {
|
||||
if k != KeyPaused {
|
||||
acts = append(acts, k)
|
||||
}
|
||||
}
|
||||
sort.Strings(acts)
|
||||
return acts
|
||||
}
|
||||
|
||||
func (m *Machine) skip(reason string) string {
|
||||
var failed []error
|
||||
if err := m.say(EventRunSkipped, map[string]any{"reason": reason}); err != nil {
|
||||
m.failure(&failed, "run.skipped", err)
|
||||
}
|
||||
return "skipped: " + reason
|
||||
}
|
||||
|
||||
// RunScheduled is the scheduled run. It answers what it did in a line; its error is a command that failed.
|
||||
func (m *Machine) RunScheduled(ctx context.Context, servedPath string) (string, error) {
|
||||
release, got, err := m.Lock(ApplyLock, false)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the lock %s: %v", ApplyLock, err)
|
||||
}
|
||||
if !got {
|
||||
return m.skip(SkipLockHeld), nil
|
||||
}
|
||||
defer release()
|
||||
text, err := m.ReadFile(ConfigFile)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s could not be read: %v", ConfigFile, err)
|
||||
}
|
||||
c, refused := ParseConfig(text, m.memMiB())
|
||||
if len(refused) > 0 {
|
||||
return "refused, nothing run: " + refused[0].Key + " " + refused[0].Reason, nil
|
||||
}
|
||||
var said []string
|
||||
var failed []error
|
||||
acts := m.handActs()
|
||||
guard := ""
|
||||
var p *PoolStatus
|
||||
if c.Pool != "" {
|
||||
wait, perr := m.poolWait(ctx, c.Pool)
|
||||
switch {
|
||||
case perr != nil:
|
||||
return "", perr
|
||||
case strings.Contains(wait, "to be imported"):
|
||||
guard = SkipPoolNotImported
|
||||
case wait != "":
|
||||
guard = SkipScanRunning
|
||||
}
|
||||
if guard == "" {
|
||||
st, err := m.Status(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
p = statusOf(st, c.Pool)
|
||||
}
|
||||
}
|
||||
for _, act := range acts {
|
||||
switch {
|
||||
case act == "apply":
|
||||
s, err := m.applyLocked(ctx)
|
||||
said = append(said, "apply: "+s)
|
||||
if err != nil {
|
||||
failed = append(failed, err)
|
||||
}
|
||||
case guard != "":
|
||||
said = append(said, m.skip(guard))
|
||||
default:
|
||||
m.Warnings = append(m.Warnings, "the hand act "+act+" is not one this run knows")
|
||||
}
|
||||
_ = m.States.Delete(StateSchedule, act)
|
||||
}
|
||||
if guard != "" || c.Pool == "" {
|
||||
if guard != "" && len(acts) == 0 {
|
||||
said = append(said, m.skip(guard))
|
||||
}
|
||||
return strings.Join(said, "; "), joinErrors(failed)
|
||||
}
|
||||
served := m.readServed(servedPath)
|
||||
now := m.Now()
|
||||
for _, work := range []struct {
|
||||
name, key, verb string
|
||||
s *Schedule
|
||||
start func() error
|
||||
}{{"scrub", c.Pool, "start", ScheduleOf(c), func() error {
|
||||
_, err := m.AsRoot(ctx, "zpool", ScrubArgv("start", c.Pool)...)
|
||||
return err
|
||||
}}, {"trim", c.Pool + "/trim", "start", TrimScheduleOf(c), func() error {
|
||||
_, err := m.AsRoot(ctx, "zpool", TrimArgv(c.Pool)...)
|
||||
return err
|
||||
}}} {
|
||||
if work.s == nil {
|
||||
continue
|
||||
}
|
||||
slot := work.s.Slot(now)
|
||||
if slot.IsZero() || served[work.key] >= slot.Unix() {
|
||||
continue
|
||||
}
|
||||
if work.name == "scrub" && startedSince(p, slot) {
|
||||
said = append(said, fmt.Sprintf("the scrub of %s scheduled %s ran already", c.Pool, slot.Format("2006-01-02 15:04")))
|
||||
} else if err := work.start(); err != nil && !strings.Contains(err.Error(), "currently") {
|
||||
failed = append(failed, fmt.Errorf("the scheduled %s of %s could not start: %v", work.name, c.Pool, err))
|
||||
continue
|
||||
} else {
|
||||
said = append(said, fmt.Sprintf("started the %s of %s scheduled %s", work.name, c.Pool, slot.Format("2006-01-02 15:04")))
|
||||
}
|
||||
served[work.key] = slot.Unix()
|
||||
if err := writeServed(servedPath, served); err != nil {
|
||||
failed = append(failed, fmt.Errorf("the %s of %s was started, and the served slot could not be kept: %v", work.name, c.Pool, err))
|
||||
}
|
||||
}
|
||||
if len(said) == 0 {
|
||||
said = append(said, "nothing due")
|
||||
}
|
||||
return strings.Join(said, "; "), joinErrors(failed)
|
||||
}
|
||||
|
||||
func joinErrors(errs []error) error {
|
||||
if len(errs) == 0 {
|
||||
return nil
|
||||
}
|
||||
var parts []string
|
||||
for _, e := range errs {
|
||||
parts = append(parts, e.Error())
|
||||
}
|
||||
return fmt.Errorf("%s", strings.Join(parts, "; "))
|
||||
}
|
||||
@@ -0,0 +1,269 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func scrubConf(pool, cadence, day, at string) string {
|
||||
return `{"pool":"` + pool + `","scrub-cadence":"` + cadence + `","scrub-day":"` + day + `","scrub-time":"` + at + `"}`
|
||||
}
|
||||
|
||||
func TestTheSlotIsTheLastScheduledTimeNotAfterNow(t *testing.T) {
|
||||
loc := time.Local
|
||||
at := func(y int, m time.Month, d, h, min int) time.Time { return time.Date(y, m, d, h, min, 0, 0, loc) }
|
||||
cases := []struct {
|
||||
conf string
|
||||
now time.Time
|
||||
want time.Time
|
||||
}{
|
||||
{scrubConf("tank", "weekly", "Mon", "00:00"), at(2026, 10, 9, 12, 0), at(2026, 10, 5, 0, 0)},
|
||||
{scrubConf("tank", "weekly", "Fri", "13:30"), at(2026, 10, 9, 12, 0), at(2026, 10, 2, 13, 30)},
|
||||
{scrubConf("tank", "weekly", "Fri", "11:30"), at(2026, 10, 9, 12, 0), at(2026, 10, 9, 11, 30)},
|
||||
{scrubConf("tank", "monthly", "Mon", "00:00"), at(2026, 10, 9, 12, 0), at(2026, 10, 5, 0, 0)},
|
||||
{scrubConf("tank", "monthly", "Mon", "00:00"), at(2026, 10, 4, 23, 0), at(2026, 9, 7, 0, 0)},
|
||||
{scrubConf("tank", "monthly", "Thu", "06:00"), at(2026, 10, 1, 6, 0), at(2026, 10, 1, 6, 0)},
|
||||
}
|
||||
for _, c := range cases {
|
||||
cfg, ref := ParseConfig([]byte(c.conf), 16000)
|
||||
if len(ref) != 0 {
|
||||
t.Fatal(ref)
|
||||
}
|
||||
s := ScheduleOf(cfg)
|
||||
if got := s.Slot(c.now); !got.Equal(c.want) {
|
||||
t.Errorf("%q at %s: slot %s, want %s", c.conf, c.now, got, c.want)
|
||||
}
|
||||
if next := s.NextSlot(c.now); !next.After(c.now) || !s.Slot(next).Equal(next) {
|
||||
t.Errorf("%q at %s: next slot %s", c.conf, c.now, next)
|
||||
}
|
||||
}
|
||||
// A schedule with a value unset is no schedule; a value of another form is refused.
|
||||
if cfg, _ := ParseConfig([]byte(`{"pool":"tank","scrub-cadence":"weekly"}`), 16000); ScheduleOf(cfg) != nil {
|
||||
t.Error("a schedule without day and time")
|
||||
}
|
||||
for _, conf := range []string{scrubConf("tank", "daily", "Mon", "00:00"), scrubConf("tank", "weekly", "Monday", "00:00"),
|
||||
scrubConf("tank", "weekly", "Mon", "25:00"), scrubConf("tank x", "weekly", "Mon", "00:00"), scrubConf("tank", "weekly", "Mon..Fri", "00:00"),
|
||||
scrubConf("tank", "weekly", "Mon", "*:*")} {
|
||||
if _, ref := ParseConfig([]byte(conf), 16000); len(ref) == 0 {
|
||||
t.Errorf("accepted %q", conf)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// runFake is a root machine on the healthy fixtures, with the config file, the schedule state and the lock.
|
||||
func runFake(t *testing.T, conf string) (*fake, *Machine, memStates, string) {
|
||||
f := withZFS(t)
|
||||
f.uid = 0
|
||||
f.files[ConfigFile] = conf
|
||||
f.files[MemInfoFile] = "MemTotal: 16384000 kB\n"
|
||||
m := f.machine()
|
||||
st := memStates{}
|
||||
m.States = st
|
||||
return f, m, st, filepath.Join(t.TempDir(), "zfs-scrub", "scrub-served.json")
|
||||
}
|
||||
|
||||
// §20, §24, §31: the lock held: nothing run; a scrub or resilver running or an unimported pool: nothing run
|
||||
// and said; a due scrub started and the run ends without waiting; a due trim by trim-day and trim-time; a
|
||||
// served slot not run again; a hand act's key acted on under the lock and deleted.
|
||||
func TestTheScheduledRunDoesWhatIsDueAndEndsWithoutWaiting(t *testing.T) {
|
||||
const trim = `{"pool":"tank","trim-schedule":"weekly","trim-day":"Mon","trim-time":"00:00"}`
|
||||
cases := []struct {
|
||||
name, conf string
|
||||
setup func(f *fake, st memStates)
|
||||
ran, not []string
|
||||
event string // event kind and reason
|
||||
served bool
|
||||
said string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "scrub due", conf: scrubConf("tank", "monthly", "Mon", "00:00"), setup: func(f *fake, _ memStates) { f.on("zpool scrub -- tank", "") },
|
||||
ran: []string{"zpool scrub -- tank"}, not: []string{"zpool wait"}, served: true, said: "started the scrub"},
|
||||
{name: "scrub refused as one just began", conf: scrubConf("tank", "monthly", "Mon", "00:00"), setup: func(f *fake, _ memStates) {
|
||||
f.answers["zpool scrub -- tank"] = Ran{Code: 1, Err: "cannot scrub tank: currently scrubbing"}
|
||||
}, ran: []string{"zpool scrub -- tank"}, not: []string{"zpool wait", "scrub -p", "scrub -s"}, served: true},
|
||||
{name: "scrub running", conf: scrubConf("tank", "monthly", "Mon", "00:00"), setup: func(f *fake, _ memStates) {
|
||||
f.on(line("zpool", StatusArgv()), fixture(t, "zpool-status-scrubbing.txt"))
|
||||
}, not: []string{"zpool scrub", "zpool wait"}, event: "run.skipped/" + SkipScanRunning},
|
||||
{name: "resilver running", conf: scrubConf("tank", "monthly", "Mon", "00:00"), setup: func(f *fake, _ memStates) {
|
||||
f.on(line("zpool", StatusArgv()), fixture(t, "zpool-status-degraded.txt"))
|
||||
}, not: []string{"zpool scrub"}, event: "run.skipped/" + SkipScanRunning},
|
||||
{name: "pool not imported", conf: scrubConf("nopool", "monthly", "Mon", "00:00"), not: []string{"zpool scrub"},
|
||||
event: "run.skipped/" + SkipPoolNotImported},
|
||||
{name: "slot already served by a scrub that started in it", conf: scrubConf("tank", "weekly", "Sun", "00:00"), setup: func(f *fake, _ memStates) {
|
||||
f.now = time.Date(2026, 9, 15, 12, 0, 0, 0, time.Local)
|
||||
}, not: []string{"zpool scrub"}, served: true, said: "ran already"},
|
||||
{name: "trim due", conf: trim, setup: func(f *fake, _ memStates) { f.on("zpool trim -- tank", "") },
|
||||
ran: []string{"zpool trim -- tank"}, served: true, said: "started the trim"},
|
||||
{name: "trim none", conf: `{"pool":"tank","trim-schedule":"none","trim-day":"Mon","trim-time":"00:00"}`, not: []string{"zpool trim"}},
|
||||
{name: "a hand act's key is acted on and deleted", conf: `{"pool":"tank"}`, setup: func(f *fake, st memStates) {
|
||||
st.Put(StateSchedule, "apply", map[string]string{"by": "x"})
|
||||
}, said: "apply: applied"},
|
||||
{name: "a refused config runs nothing", conf: `{"pool":"tank","scrub-cadence":"hourly"}`, not: []string{"zpool"}, said: "refused"},
|
||||
{name: "a failed start fails the run", conf: scrubConf("tank", "monthly", "Mon", "00:00"), setup: func(f *fake, _ memStates) {
|
||||
f.answers["zpool scrub -- tank"] = Ran{Code: 1, Err: "cannot scrub: no such pool"}
|
||||
}, ran: []string{"zpool scrub -- tank"}, wantErr: true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
f, m, st, served := runFake(t, c.conf)
|
||||
if c.setup != nil {
|
||||
c.setup(f, st)
|
||||
}
|
||||
said, err := m.RunScheduled(context.Background(), served)
|
||||
if (err != nil) != c.wantErr {
|
||||
t.Errorf("%s: error %v", c.name, err)
|
||||
}
|
||||
for _, r := range c.ran {
|
||||
if len(ran(f, r)) != 1 {
|
||||
t.Errorf("%s: %q not run once; ran %v", c.name, r, f.calls)
|
||||
}
|
||||
}
|
||||
for _, n := range c.not {
|
||||
if len(ran(f, n)) != 0 {
|
||||
t.Errorf("%s: ran %v", c.name, ran(f, n))
|
||||
}
|
||||
}
|
||||
if c.event != "" {
|
||||
kind, reason, _ := strings.Cut(c.event, "/")
|
||||
if len(f.emitted) != 1 || f.emitted[0]["_type"] != kind || f.emitted[0]["reason"] != reason {
|
||||
t.Errorf("%s: events %v, want %s", c.name, f.emitted, c.event)
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(served); (err == nil) != c.served {
|
||||
t.Errorf("%s: served file present %v, want %v", c.name, err == nil, c.served)
|
||||
}
|
||||
if !strings.Contains(said, c.said) {
|
||||
t.Errorf("%s: said %q, want %q", c.name, said, c.said)
|
||||
}
|
||||
if _, ok := st["schedule/apply"]; ok {
|
||||
t.Errorf("%s: the hand act's key stayed", c.name)
|
||||
}
|
||||
}
|
||||
// The lock held by another: nothing run, said.
|
||||
f, m, _, served := runFake(t, scrubConf("tank", "monthly", "Mon", "00:00"))
|
||||
m.Lock = func(string, bool) (func(), bool, error) { return nil, false, nil }
|
||||
if said, err := m.RunScheduled(context.Background(), served); err != nil || !strings.Contains(said, SkipLockHeld) ||
|
||||
len(f.emitted) != 1 || f.emitted[0]["reason"] != SkipLockHeld || len(ran(f, "zpool")) != 0 {
|
||||
t.Errorf("lock held: %q %v %v %v", said, err, f.emitted, f.calls)
|
||||
}
|
||||
}
|
||||
|
||||
// scheduled is the healthy fixtures with the module's scrub schedule for tank, its timer waiting and its
|
||||
// last run a success.
|
||||
func scheduled(t *testing.T) *fake {
|
||||
f := withZFS(t)
|
||||
f.files[ConfigFile] = scrubConf("tank", "weekly", "Mon", "00:00")
|
||||
f.files[MemInfoFile] = "MemTotal: 16384000 kB\n"
|
||||
f.on(line("systemctl", TimerShowArgv(RunTimer)),
|
||||
"ActiveState=active\nUnitFileState=enabled\nNextElapseUSecRealtime=@1791842400\nLastTriggerUSec=@1791237600\n")
|
||||
f.on(line("systemctl", ServiceShowArgv(RunService)),
|
||||
"ActiveState=inactive\nResult=success\nExecMainStartTimestamp=@1791237600\nExecMainExitTimestamp=@1791331200\n")
|
||||
return f
|
||||
}
|
||||
|
||||
func TestTheScheduleIsTheConfigAndWhatTheTimerSays(t *testing.T) {
|
||||
f := scheduled(t)
|
||||
got, err := f.machine().Scrubs(context.Background(), false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := got.(ScrubAnswer)
|
||||
s := a.Schedule
|
||||
if s == nil || s.Pool != "tank" || s.Cadence != "weekly" || s.Timer != "zfs-mesh-run.timer" || s.TimerState != "active" ||
|
||||
s.Enabled != "enabled" || s.Next == nil || !s.Next.Equal(time.Date(2026, 10, 12, 0, 0, 0, 0, time.Local)) || s.LastResult != "success" {
|
||||
t.Fatalf("schedule: %+v", s)
|
||||
}
|
||||
f = withZFS(t)
|
||||
got, _ = f.machine().Scrubs(context.Background(), false)
|
||||
if a := got.(ScrubAnswer); a.Schedule != nil || !strings.Contains(strings.Join(a.Said, " "), "no scrub schedule") {
|
||||
t.Errorf("no config: %+v", a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheScheduleWarnsOfAnOldScrubAFailedRunAndAStoppedTimer(t *testing.T) {
|
||||
f := scheduled(t)
|
||||
h := f.machine().Health(context.Background())
|
||||
if !h.Healthy {
|
||||
t.Fatalf("%+v", h)
|
||||
}
|
||||
w := strings.Join(h.Warnings, " ")
|
||||
// tank's last scrub finished 2026-09-14, 25 days before the fake clock: past weekly plus its margin.
|
||||
if !strings.Contains(w, "Pool tank was last scrubbed 25 days ago, more than its weekly schedule allows (9 days)") ||
|
||||
!strings.Contains(w, "The scheduled scrub of tank due 2026-10-05 00:00 has not started") {
|
||||
t.Errorf("warnings: %v", h.Warnings)
|
||||
}
|
||||
f.on(line("systemctl", ServiceShowArgv(RunService)),
|
||||
"ActiveState=failed\nResult=exit-code\nExecMainStartTimestamp=@1791237600\nExecMainExitTimestamp=@1791237601\n")
|
||||
f.on(line("systemctl", TimerShowArgv(RunTimer)),
|
||||
"ActiveState=inactive\nUnitFileState=disabled\nNextElapseUSecRealtime=\nLastTriggerUSec=@1791237600\n")
|
||||
w = strings.Join(f.machine().Health(context.Background()).Warnings, " ")
|
||||
for _, want := range []string{"The last scheduled scrub of tank failed (exit-code)", "The scrub schedule's timer zfs-mesh-run.timer is inactive and disabled"} {
|
||||
if !strings.Contains(w, want) {
|
||||
t.Errorf("lacks %q: %s", want, w)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheScrubActsAreFixedCommandsWithWhyReadBackAndRecorded(t *testing.T) {
|
||||
for act, cmd := range map[string]string{
|
||||
"start": "sudo -n zpool scrub -- tank",
|
||||
"pause": "sudo -n zpool scrub -p -- tank",
|
||||
"stop": "sudo -n zpool scrub -s -- tank",
|
||||
} {
|
||||
f := withZFS(t)
|
||||
f.on(cmd, "")
|
||||
a, err := f.machine().ScrubAct(context.Background(), act, "tank", "the operator asked")
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", act, err)
|
||||
}
|
||||
if strings.Join(a.Ran, " ") != cmd || a.After == "" || !a.Done {
|
||||
t.Errorf("%s: %+v", act, a)
|
||||
}
|
||||
if len(f.emitted) != 1 || f.emitted[0]["_type"] != ScrubEvent || f.emitted[0]["act"] != act || f.emitted[0]["why"] != "the operator asked" {
|
||||
t.Errorf("%s recorded: %+v", act, f.emitted)
|
||||
}
|
||||
}
|
||||
f := withZFS(t)
|
||||
for _, c := range []struct{ act, pool, why, says string }{
|
||||
{"start", "nope", "why", "no pool"},
|
||||
{"start", "tank", "", "say why"},
|
||||
{"start", "-s", "why", "no pool"},
|
||||
{"destroy", "tank", "why", "not an act"},
|
||||
} {
|
||||
if _, err := f.machine().ScrubAct(context.Background(), c.act, c.pool, c.why); err == nil || !strings.Contains(err.Error(), c.says) {
|
||||
t.Errorf("%+v: %v", c, err)
|
||||
}
|
||||
}
|
||||
if n := len(ran(f, "sudo")); n != 0 {
|
||||
t.Errorf("a refusal ran %v", f.calls)
|
||||
}
|
||||
f.answers["sudo -n zpool scrub -- tank"] = Ran{Code: -1, TimedOut: true}
|
||||
if _, err := f.machine().ScrubAct(context.Background(), "start", "tank", "why"); err == nil || !strings.Contains(err.Error(), "outcome unknown; check zfs_scrub") {
|
||||
t.Errorf("timed out: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Third review: the scheduled run's state is root's alone — a folder of its own, 0700 — so the operator account, or anything it runs, cannot make the run write elsewhere.
|
||||
func TestTheServedSlotIsWrittenOnlyIntoAPrivateFolderAndNeverThroughALink(t *testing.T) {
|
||||
dir := filepath.Join(t.TempDir(), "zfs-scrub")
|
||||
path := filepath.Join(dir, "scrub-served.json")
|
||||
if err := writeServed(path, map[string]int64{"tank": 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fi, err := os.Lstat(dir); err != nil || fi.Mode().Perm() != 0o700 || !fi.IsDir() {
|
||||
t.Errorf("the folder: %v %v", fi.Mode(), err)
|
||||
}
|
||||
// A folder open to others: refused. (The refusal of a link is not run here: no test of this module makes a
|
||||
// symbolic link, the mesh's hard rule.)
|
||||
open := filepath.Join(t.TempDir(), "open")
|
||||
os.Mkdir(open, 0o755)
|
||||
if err := writeServed(filepath.Join(open, "scrub-served.json"), map[string]int64{"tank": 3}); err == nil {
|
||||
t.Errorf("written into a folder others can read")
|
||||
}
|
||||
if filepath.Dir(ServedFile) == filepath.Dir(ErrorsFile) {
|
||||
t.Errorf("the run's state shares the account's folder")
|
||||
}
|
||||
}
|
||||
@@ -15,21 +15,19 @@ import (
|
||||
|
||||
// The scrub schedule the module owns.
|
||||
//
|
||||
// The mesh writes ScrubConfig from the assignment's settings. The node-engine runs the module's
|
||||
// scheduled process, `zfs-tools scrub-run`, from its own timer (ScrubTimer) every quarter of an hour, and
|
||||
// the run decides from the configuration whether a scrub is due: the last scheduled time not after now
|
||||
// (the slot) not yet served. Due, it starts the pool's scrub, or waits for one already running, as
|
||||
// upstream's zfs-scrub@.service does; it never pauses or stops one, and nothing stops the unit to pause
|
||||
// it (the node-engine writes no ExecStop). So no setting value ever reaches a unit file: the day and the
|
||||
// time are read and checked here, and an invalid one is refused before anything runs.
|
||||
// The node-engine writes ConfigFile from the config values. The node-engine runs the module's scheduled
|
||||
// process, `zfs-tools run`, from its own timer (RunTimer) every quarter of an hour, and the run decides from the
|
||||
// config whether a scrub is due: the last scheduled time not after now (the slot) not yet served. Due, it starts
|
||||
// the pool's scrub and returns; it never pauses or stops one and never waits for one (run.go). The day and the
|
||||
// time are read and checked by the parse (config.go), so no setting value ever reaches a unit file.
|
||||
//
|
||||
// The mesh holds the distribution's own scrub timers for the pool stopped and disabled, so one schedule
|
||||
// scrubs it; the node-engine gives them back running and enabled when the module goes (novox/hq ADR 0269).
|
||||
|
||||
// ScrubTimer and ScrubService are the node-engine's units for the scheduled run (the process's name).
|
||||
// RunTimer and RunService are the node-engine's units for the scheduled run (the process's name).
|
||||
const (
|
||||
ScrubTimer = "zfs-mesh-scrub.timer"
|
||||
ScrubService = "zfs-mesh-scrub.service"
|
||||
RunTimer = "zfs-mesh-run.timer"
|
||||
RunService = "zfs-mesh-run.service"
|
||||
)
|
||||
|
||||
// ServedFile is where the scheduled run keeps the last slot it served, per pool: in a folder of root's
|
||||
@@ -42,9 +40,6 @@ const ServedFile = "/var/lib/zfs-scrub/scrub-served.json"
|
||||
// fires every quarter of an hour.
|
||||
const MissedAfter = time.Hour
|
||||
|
||||
// ScrubConfig is the schedule as the mesh wrote it from the settings.
|
||||
const ScrubConfig = "/etc/zfs-tools/scrub.conf"
|
||||
|
||||
// Margins after a cadence before a last scrub is said to be late: a weekly run that waited for a busy
|
||||
// pool, a monthly run on the first given weekday of a month (28 or 35 days apart).
|
||||
var cadenceAllows = map[string]time.Duration{
|
||||
@@ -81,33 +76,22 @@ var (
|
||||
clock = regexp.MustCompile(`^([01][0-9]|2[0-3]):[0-5][0-9]$`)
|
||||
)
|
||||
|
||||
// ParseScrubConfig reads the schedule: pool, cadence (weekly or monthly), day (Mon … Sun) and time (HH:MM).
|
||||
func ParseScrubConfig(text string) (*Schedule, error) {
|
||||
v := map[string]string{}
|
||||
for _, l := range strings.Split(text, "\n") {
|
||||
l = strings.TrimSpace(l)
|
||||
if l == "" || strings.HasPrefix(l, "#") {
|
||||
continue
|
||||
}
|
||||
k, val, ok := strings.Cut(l, "=")
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%s: %q is not key=value", ScrubConfig, l)
|
||||
}
|
||||
v[strings.TrimSpace(k)] = strings.TrimSpace(val)
|
||||
// ScheduleOf is the scrub schedule the config holds: the pool, and the cadence, day and time. nil when the
|
||||
// values are not all set (the module then has no scrub schedule, §11: not managed until set).
|
||||
func ScheduleOf(c *Config) *Schedule {
|
||||
if c.Pool == "" || c.ScrubCadence == "" || c.ScrubDay == "" || c.ScrubTime == "" {
|
||||
return nil
|
||||
}
|
||||
s := &Schedule{Pool: v["pool"], Cadence: v["cadence"], Day: v["day"], Time: v["time"]}
|
||||
switch {
|
||||
case !poolName.MatchString(s.Pool):
|
||||
return nil, fmt.Errorf("the scrub-pool setting %q is not a pool's name", s.Pool)
|
||||
case cadenceAllows[s.Cadence] == 0:
|
||||
return nil, fmt.Errorf("the scrub-cadence setting is %q; it is weekly or monthly", s.Cadence)
|
||||
case !weekday.MatchString(s.Day):
|
||||
return nil, fmt.Errorf("the scrub-day setting is %q; it is Mon, Tue, Wed, Thu, Fri, Sat or Sun", s.Day)
|
||||
case !clock.MatchString(s.Time):
|
||||
return nil, fmt.Errorf("the scrub-time setting is %q; it is HH:MM, 00:00 to 23:59", s.Time)
|
||||
return &Schedule{Pool: c.Pool, Cadence: c.ScrubCadence, Day: c.ScrubDay, Time: c.ScrubTime, Timer: RunTimer, Service: RunService}
|
||||
}
|
||||
|
||||
// TrimScheduleOf is the trim schedule: nil while trim-schedule is not weekly or monthly, or its day or time is
|
||||
// not set.
|
||||
func TrimScheduleOf(c *Config) *Schedule {
|
||||
if c.Pool == "" || (c.TrimSchedule != "weekly" && c.TrimSchedule != "monthly") || c.TrimDay == "" || c.TrimTime == "" {
|
||||
return nil
|
||||
}
|
||||
s.Timer, s.Service = ScrubTimer, ScrubService
|
||||
return s, nil
|
||||
return &Schedule{Pool: c.Pool, Cadence: c.TrimSchedule, Day: c.TrimDay, Time: c.TrimTime, Timer: RunTimer, Service: RunService}
|
||||
}
|
||||
|
||||
var weekdays = map[string]time.Weekday{"Sun": time.Sunday, "Mon": time.Monday, "Tue": time.Tuesday,
|
||||
@@ -150,13 +134,6 @@ func (s *Schedule) NextSlot(now time.Time) time.Time {
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
// WaitArgv waits for a pool's running scrub to end.
|
||||
func WaitArgv(pool string) []string { return []string{"wait", "-t", "scrub", "--", pool} }
|
||||
|
||||
// WaitBound is how long the scheduled run waits for a scrub: past it the run ends and leaves the scrub
|
||||
// running; a scrub never pauses for it.
|
||||
const WaitBound = 7 * 24 * time.Hour
|
||||
|
||||
// readServed is the last slot served per pool; none when the file is not there or not readable.
|
||||
func (m *Machine) readServed(path string) map[string]int64 {
|
||||
out := map[string]int64{}
|
||||
@@ -220,72 +197,6 @@ func startedSince(p *PoolStatus, slot time.Time) bool {
|
||||
return sc != nil && sc.Started != nil && !sc.Started.Before(slot)
|
||||
}
|
||||
|
||||
// ScrubRun is the scheduled run: a due scrub started, or one running waited for. It answers what it did in
|
||||
// a line, and fails on an invalid schedule, which it never acts on.
|
||||
func (m *Machine) ScrubRun(ctx context.Context, servedPath string) (string, error) {
|
||||
raw, err := m.ReadFile(ScrubConfig)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s could not be read: %v", ScrubConfig, err)
|
||||
}
|
||||
s, err := ParseScrubConfig(string(raw))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("refused, nothing run: %v", err)
|
||||
}
|
||||
now := m.Now()
|
||||
slot := s.Slot(now)
|
||||
served := m.readServed(servedPath)
|
||||
if served[s.Pool] >= slot.Unix() {
|
||||
return fmt.Sprintf("the scrub of %s scheduled %s was served", s.Pool, slot.Format("2006-01-02 15:04")), nil
|
||||
}
|
||||
st, err := m.Status(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
p := statusOf(st, s.Pool)
|
||||
if p == nil {
|
||||
return "", fmt.Errorf("the scheduled pool %s is not imported here", s.Pool)
|
||||
}
|
||||
mark := func() error {
|
||||
served[s.Pool] = slot.Unix()
|
||||
return writeServed(servedPath, served)
|
||||
}
|
||||
if startedSince(p, slot) && p.Scrub().State != "running" {
|
||||
return fmt.Sprintf("the scrub of %s scheduled %s ran already", s.Pool, slot.Format("2006-01-02 15:04")), mark()
|
||||
}
|
||||
for _, r := range p.Resilvers() {
|
||||
if r.State == "running" {
|
||||
return fmt.Sprintf("a resilver of %s runs; the scrub scheduled %s waits for the next run after it", s.Pool,
|
||||
slot.Format("2006-01-02 15:04")), nil
|
||||
}
|
||||
}
|
||||
started := false
|
||||
if sc := p.Scrub(); sc == nil || sc.State != "running" {
|
||||
_, err := m.AsRoot(ctx, "zpool", ScrubArgv("start", s.Pool)...)
|
||||
switch {
|
||||
case err == nil:
|
||||
started = true
|
||||
case strings.Contains(err.Error(), "currently scrubbing"):
|
||||
// One began since status was read: it serves the slot, and is waited for like any other.
|
||||
default:
|
||||
return "", fmt.Errorf("the scheduled scrub of %s could not start: %v", s.Pool, err)
|
||||
}
|
||||
}
|
||||
if err := mark(); err != nil {
|
||||
return "", fmt.Errorf("the scrub of %s runs, and the served slot could not be kept: %v", s.Pool, err)
|
||||
}
|
||||
wctx, cancel := context.WithTimeout(ctx, WaitBound)
|
||||
defer cancel()
|
||||
_, werr := m.AsRoot(wctx, "zpool", WaitArgv(s.Pool)...)
|
||||
what := "waited for the scrub of " + s.Pool + " already running"
|
||||
if started {
|
||||
what = "started the scrub of " + s.Pool + " scheduled " + slot.Format("2006-01-02 15:04") + ", and waited for it"
|
||||
}
|
||||
if werr != nil {
|
||||
return what + "; the wait ended without its end (" + werr.Error() + "), and the scrub goes on", nil
|
||||
}
|
||||
return what + " to finish", nil
|
||||
}
|
||||
|
||||
// TimerShowArgv asks the service manager what a timer says: whether it runs and is enabled, when it
|
||||
// fires next and when it last did, as Unix times.
|
||||
func TimerShowArgv(unit string) []string {
|
||||
@@ -320,15 +231,19 @@ func unixStamp(s string) *time.Time {
|
||||
}
|
||||
|
||||
// ScheduleNow is the schedule with what the service manager says of it; nil when the module has no
|
||||
// schedule on this machine (no configuration written).
|
||||
// schedule on this machine (no config file, or no cadence, day and time set).
|
||||
func (m *Machine) ScheduleNow(ctx context.Context) (*Schedule, error) {
|
||||
raw, err := m.ReadFile(ScrubConfig)
|
||||
raw, err := m.ReadFile(ConfigFile)
|
||||
if err != nil {
|
||||
return nil, nil
|
||||
}
|
||||
s, err := ParseScrubConfig(string(raw))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
c, refused := ParseConfig(raw, m.memMiB())
|
||||
if len(refused) > 0 {
|
||||
return nil, fmt.Errorf("%s refused %s %s", ConfigFile, refused[0].Key, refused[0].Reason)
|
||||
}
|
||||
s := ScheduleOf(c)
|
||||
if s == nil {
|
||||
return nil, nil
|
||||
}
|
||||
if out, err := m.ask(ctx, "systemctl", TimerShowArgv(s.Timer)...); err != nil {
|
||||
s.Note = "the service manager did not say what the timer is: " + err.Error()
|
||||
|
||||
@@ -141,7 +141,7 @@ func (m *Machine) Scrubs(ctx context.Context, history bool) (any, error) {
|
||||
case serr != nil:
|
||||
a.Said = append(a.Said, "The scrub schedule's settings are not valid: "+serr.Error()+".")
|
||||
case sched == nil:
|
||||
a.Said = append(a.Said, "The module has no scrub schedule on this machine (no "+ScrubConfig+").")
|
||||
a.Said = append(a.Said, "The module has no scrub schedule on this machine (no cadence, day and time set in "+ConfigFile+").")
|
||||
default:
|
||||
a.Schedule = sched
|
||||
line := fmt.Sprintf("%s scrub of %s, %s at %s (monthly: the first %s of the month), decided by the run %s fires "+
|
||||
|
||||
@@ -1,206 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The scheduled run (`zfs-tools scrub-run`, second review of mesh-catalog #147): the node-engine's timer
|
||||
// fires it every quarter of an hour, and it decides from the configuration whether a scrub is due. It
|
||||
// starts one, or waits for one already running, as upstream's zfs-scrub@.service does, and never pauses or
|
||||
// stops a scrub.
|
||||
|
||||
func monthlyConf(pool string) string {
|
||||
return "pool=" + pool + "\ncadence=monthly\nday=Mon\ntime=00:00\n"
|
||||
}
|
||||
|
||||
func TestTheSlotIsTheLastScheduledTimeNotAfterNow(t *testing.T) {
|
||||
loc := time.Local
|
||||
at := func(y int, m time.Month, d, h, min int) time.Time { return time.Date(y, m, d, h, min, 0, 0, loc) }
|
||||
cases := []struct {
|
||||
conf string
|
||||
now time.Time
|
||||
want time.Time
|
||||
}{
|
||||
{"pool=tank\ncadence=weekly\nday=Mon\ntime=00:00\n", at(2026, 10, 9, 12, 0), at(2026, 10, 5, 0, 0)},
|
||||
{"pool=tank\ncadence=weekly\nday=Fri\ntime=13:30\n", at(2026, 10, 9, 12, 0), at(2026, 10, 2, 13, 30)},
|
||||
{"pool=tank\ncadence=weekly\nday=Fri\ntime=11:30\n", at(2026, 10, 9, 12, 0), at(2026, 10, 9, 11, 30)},
|
||||
{monthlyConf("tank"), at(2026, 10, 9, 12, 0), at(2026, 10, 5, 0, 0)},
|
||||
{monthlyConf("tank"), at(2026, 10, 4, 23, 0), at(2026, 9, 7, 0, 0)},
|
||||
{"pool=tank\ncadence=monthly\nday=Thu\ntime=06:00\n", at(2026, 10, 1, 6, 0), at(2026, 10, 1, 6, 0)},
|
||||
}
|
||||
for _, c := range cases {
|
||||
s, err := ParseScrubConfig(c.conf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := s.Slot(c.now); !got.Equal(c.want) {
|
||||
t.Errorf("%q at %s: slot %s, want %s", c.conf, c.now, got, c.want)
|
||||
}
|
||||
if next := s.NextSlot(c.now); !next.After(c.now) || !s.Slot(next).Equal(next) {
|
||||
t.Errorf("%q at %s: next slot %s", c.conf, c.now, next)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// root runs the scheduled run: no sudo.
|
||||
func rootFake(t *testing.T, conf string) (*fake, string) {
|
||||
f := withZFS(t)
|
||||
f.uid = 0
|
||||
f.files[ScrubConfig] = conf
|
||||
dir := filepath.Join(t.TempDir(), "zfs-scrub")
|
||||
return f, filepath.Join(dir, "scrub-served.json")
|
||||
}
|
||||
|
||||
func TestADueScrubIsStartedThenWaitedForAndMarkedServed(t *testing.T) {
|
||||
f, served := rootFake(t, monthlyConf("tank"))
|
||||
f.on("zpool scrub -- tank", "")
|
||||
f.on("zpool wait -t scrub -- tank", "")
|
||||
said, err := f.machine().ScrubRun(context.Background(), served)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := strings.Join(ran(f, "zpool scrub"), "|") + "/" + strings.Join(ran(f, "zpool wait"), "|"); got != "zpool scrub -- tank/zpool wait -t scrub -- tank" {
|
||||
t.Errorf("ran %q", got)
|
||||
}
|
||||
if !strings.Contains(said, "started") {
|
||||
t.Errorf("said %q", said)
|
||||
}
|
||||
// Served: the next fire within the same slot runs nothing.
|
||||
f.calls = nil
|
||||
if _, err := f.machine().ScrubRun(context.Background(), served); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := len(ran(f, "zpool scrub")) + len(ran(f, "zpool wait")); n != 0 {
|
||||
t.Errorf("a served slot ran %v", f.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAScrubInProgressIsWaitedForNeverPausedOrStarted(t *testing.T) {
|
||||
f, served := rootFake(t, monthlyConf("tank"))
|
||||
f.on(line("zpool", StatusArgv()), fixture(t, "zpool-status-scrubbing.txt"))
|
||||
f.on(line("zpool", PoolsArgv()), "tank\t1000\t100\t900\t10\t1\tONLINE\t1.00\n")
|
||||
f.on("zpool wait -t scrub -- tank", "")
|
||||
if _, err := f.machine().ScrubRun(context.Background(), served); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Join(f.calls, "|") == "" || len(ran(f, "zpool scrub")) != 0 || len(ran(f, "zpool wait -t scrub -- tank")) != 1 {
|
||||
t.Errorf("ran %v", f.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStartRefusedBecauseAScrubJustBeganIsWaitedForNotPaused(t *testing.T) {
|
||||
f, served := rootFake(t, monthlyConf("tank"))
|
||||
f.answers["zpool scrub -- tank"] = Ran{Code: 1, Err: "cannot scrub tank: currently scrubbing; use 'zpool scrub -s' to cancel the current scrub"}
|
||||
f.on("zpool wait -t scrub -- tank", "")
|
||||
if _, err := f.machine().ScrubRun(context.Background(), served); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range f.calls {
|
||||
if strings.Contains(c, "scrub -p") || strings.Contains(c, "scrub -s") {
|
||||
t.Errorf("the run paused or stopped a scrub: %s", c)
|
||||
}
|
||||
}
|
||||
if len(ran(f, "zpool wait -t scrub -- tank")) != 1 {
|
||||
t.Errorf("not waited for: %v", f.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAResilverDefersTheScrubWithoutServingTheSlot(t *testing.T) {
|
||||
f, served := rootFake(t, monthlyConf("tank"))
|
||||
f.on(line("zpool", StatusArgv()), fixture(t, "zpool-status-degraded.txt"))
|
||||
f.on(line("zpool", PoolsArgv()), "tank\t1000\t100\t900\t10\t1\tDEGRADED\t1.00\n")
|
||||
said, err := f.machine().ScrubRun(context.Background(), served)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(ran(f, "zpool scrub")) != 0 || !strings.Contains(said, "resilver") {
|
||||
t.Errorf("ran %v, said %q", f.calls, said)
|
||||
}
|
||||
if _, err := os.Stat(served); err == nil {
|
||||
t.Errorf("the slot was marked served while the scrub waits for the resilver")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAScrubAlreadyStartedInTheSlotServesItWithoutAStateFile(t *testing.T) {
|
||||
// tank's scrub started 2026-09-13 00:24 (finished 09-14); a weekly Sunday-00:00 slot of 09-13 is served
|
||||
// by it, so a run on 09-15 starts nothing.
|
||||
f, served := rootFake(t, "pool=tank\ncadence=weekly\nday=Sun\ntime=00:00\n")
|
||||
f.now = time.Date(2026, 9, 15, 12, 0, 0, 0, time.Local)
|
||||
if _, err := f.machine().ScrubRun(context.Background(), served); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := len(ran(f, "zpool scrub")) + len(ran(f, "zpool wait")); n != 0 {
|
||||
t.Errorf("ran %v", f.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnInvalidScheduleRunsNothingAndFails(t *testing.T) {
|
||||
for _, conf := range []string{
|
||||
"pool=tank\ncadence=monthly\nday=Mon\ntime=*:*\n",
|
||||
"pool=tank\ncadence=monthly\nday=Mon..Fri\ntime=00:00\n",
|
||||
"pool=tank\ncadence=monthly\nday=*\ntime=00:00\n",
|
||||
"pool=tank\ncadence=hourly\nday=Mon\ntime=00:00\n",
|
||||
"pool=tank\ncadence=monthly\nday=Mon\ntime=0:00\n",
|
||||
} {
|
||||
f, served := rootFake(t, conf)
|
||||
if _, err := f.machine().ScrubRun(context.Background(), served); err == nil {
|
||||
t.Errorf("%q: not refused", conf)
|
||||
}
|
||||
if len(ran(f, "zpool scrub"))+len(ran(f, "zpool wait")) != 0 {
|
||||
t.Errorf("%q ran %v", conf, f.calls)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAScheduledScrubThatDidNotStartIsAWarning(t *testing.T) {
|
||||
f := withZFS(t)
|
||||
f.files[ScrubConfig] = "pool=tank\ncadence=weekly\nday=Mon\ntime=00:00\n"
|
||||
f.on(line("systemctl", TimerShowArgv(ScrubTimer)), "ActiveState=active\nUnitFileState=enabled\nNextElapseUSecRealtime=@1791842400\nLastTriggerUSec=@1791237600\n")
|
||||
h := f.machine().Health(context.Background())
|
||||
if !strings.Contains(strings.Join(h.Warnings, " "), "The scheduled scrub of tank due 2026-10-05 00:00 has not started") {
|
||||
t.Errorf("warnings: %v", h.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
// Third review: the scheduled run's state is root's alone — a folder of its own, 0700, never through a
|
||||
// link — so the operator account, or anything it runs, cannot make the run write elsewhere.
|
||||
func TestTheServedSlotIsWrittenOnlyIntoAPrivateFolderAndNeverThroughALink(t *testing.T) {
|
||||
dir := filepath.Join(t.TempDir(), "zfs-scrub")
|
||||
path := filepath.Join(dir, "scrub-served.json")
|
||||
if err := writeServed(path, map[string]int64{"tank": 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fi, err := os.Lstat(dir); err != nil || fi.Mode().Perm() != 0o700 || !fi.IsDir() {
|
||||
t.Errorf("the folder: %v %v", fi.Mode(), err)
|
||||
}
|
||||
// The file replaced by a link to somewhere else: refused, and the other place untouched.
|
||||
victim := filepath.Join(t.TempDir(), "victim")
|
||||
os.WriteFile(victim, []byte("keep"), 0o644)
|
||||
os.Remove(path)
|
||||
os.Symlink(victim, path)
|
||||
if err := writeServed(path, map[string]int64{"tank": 2}); err == nil {
|
||||
t.Errorf("written through a link")
|
||||
}
|
||||
if b, _ := os.ReadFile(victim); string(b) != "keep" {
|
||||
t.Errorf("the link's target was changed: %q", b)
|
||||
}
|
||||
// The folder itself a link, or open to others: refused.
|
||||
linked := filepath.Join(t.TempDir(), "linked")
|
||||
os.Symlink(t.TempDir(), linked)
|
||||
if err := writeServed(filepath.Join(linked, "scrub-served.json"), map[string]int64{"tank": 3}); err == nil {
|
||||
t.Errorf("written into a linked folder")
|
||||
}
|
||||
open := filepath.Join(t.TempDir(), "open")
|
||||
os.Mkdir(open, 0o755)
|
||||
if err := writeServed(filepath.Join(open, "scrub-served.json"), map[string]int64{"tank": 3}); err == nil {
|
||||
t.Errorf("written into a folder others can read")
|
||||
}
|
||||
if filepath.Dir(ServedFile) == filepath.Dir(ErrorsFile) {
|
||||
t.Errorf("the run's state shares the account's folder")
|
||||
}
|
||||
}
|
||||
@@ -1,119 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
const scrubConf = "pool=tank\ncadence=weekly\nday=Mon\ntime=00:00\n"
|
||||
|
||||
// scheduled is the healthy fixtures with the module's scrub schedule for tank, its timer waiting and its
|
||||
// last run a success.
|
||||
func scheduled(t *testing.T) *fake {
|
||||
f := withZFS(t)
|
||||
f.files[ScrubConfig] = scrubConf
|
||||
f.on(line("systemctl", TimerShowArgv(ScrubTimer)),
|
||||
"ActiveState=active\nUnitFileState=enabled\nNextElapseUSecRealtime=@1791842400\nLastTriggerUSec=@1791237600\n")
|
||||
f.on(line("systemctl", ServiceShowArgv(ScrubService)),
|
||||
"ActiveState=inactive\nResult=success\nExecMainStartTimestamp=@1791237600\nExecMainExitTimestamp=@1791331200\n")
|
||||
return f
|
||||
}
|
||||
|
||||
func TestTheScheduleIsTheConfigAndWhatTheTimerSays(t *testing.T) {
|
||||
f := scheduled(t)
|
||||
got, err := f.machine().Scrubs(context.Background(), false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := got.(ScrubAnswer)
|
||||
s := a.Schedule
|
||||
if s == nil || s.Pool != "tank" || s.Cadence != "weekly" || s.Timer != "zfs-mesh-scrub.timer" || s.TimerState != "active" ||
|
||||
s.Enabled != "enabled" || s.Next == nil || !s.Next.Equal(time.Date(2026, 10, 12, 0, 0, 0, 0, time.Local)) || s.LastResult != "success" {
|
||||
t.Fatalf("schedule: %+v", s)
|
||||
}
|
||||
if want := s.NextSlot(f.now).Format("2006-01-02 15:04"); !strings.Contains(strings.Join(a.Said, "\n"), "next scheduled scrub of tank: "+want) || want != "2026-10-12 00:00" {
|
||||
t.Errorf("said: %v", a.Said)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAScheduleThatIsNotSetIsSaidAndAnInvalidOneRefused(t *testing.T) {
|
||||
f := withZFS(t)
|
||||
got, _ := f.machine().Scrubs(context.Background(), false)
|
||||
if a := got.(ScrubAnswer); a.Schedule != nil || !strings.Contains(strings.Join(a.Said, " "), "no scrub schedule") {
|
||||
t.Errorf("no config: %+v", a)
|
||||
}
|
||||
for _, conf := range []string{
|
||||
"pool=tank\ncadence=daily\nday=Mon\ntime=00:00\n",
|
||||
"pool=tank\ncadence=weekly\nday=Monday\ntime=00:00\n",
|
||||
"pool=tank\ncadence=weekly\nday=Mon\ntime=25:00\n",
|
||||
"pool=tank x\ncadence=weekly\nday=Mon\ntime=00:00\n",
|
||||
} {
|
||||
if _, err := ParseScrubConfig(conf); err == nil {
|
||||
t.Errorf("accepted %q", conf)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheScheduleWarnsOfAnOldScrubAFailedRunAndAStoppedTimer(t *testing.T) {
|
||||
f := scheduled(t)
|
||||
h := f.machine().Health(context.Background())
|
||||
if !h.Healthy {
|
||||
t.Fatalf("%+v", h)
|
||||
}
|
||||
w := strings.Join(h.Warnings, " ")
|
||||
// tank's last scrub finished 2026-09-14, 25 days before the fake clock: past weekly plus its margin.
|
||||
if !strings.Contains(w, "Pool tank was last scrubbed 25 days ago, more than its weekly schedule allows (9 days)") {
|
||||
t.Errorf("warnings: %v", h.Warnings)
|
||||
}
|
||||
f.on(line("systemctl", ServiceShowArgv(ScrubService)),
|
||||
"ActiveState=failed\nResult=exit-code\nExecMainStartTimestamp=@1791237600\nExecMainExitTimestamp=@1791237601\n")
|
||||
f.on(line("systemctl", TimerShowArgv(ScrubTimer)),
|
||||
"ActiveState=inactive\nUnitFileState=disabled\nNextElapseUSecRealtime=\nLastTriggerUSec=@1791237600\n")
|
||||
w = strings.Join(f.machine().Health(context.Background()).Warnings, " ")
|
||||
for _, want := range []string{"The last scheduled scrub of tank failed (exit-code)", "The scrub schedule's timer zfs-mesh-scrub.timer is inactive and disabled"} {
|
||||
if !strings.Contains(w, want) {
|
||||
t.Errorf("lacks %q: %s", want, w)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheScrubActsAreFixedCommandsWithWhyReadBackAndRecorded(t *testing.T) {
|
||||
for act, cmd := range map[string]string{
|
||||
"start": "sudo -n zpool scrub -- tank",
|
||||
"pause": "sudo -n zpool scrub -p -- tank",
|
||||
"stop": "sudo -n zpool scrub -s -- tank",
|
||||
} {
|
||||
f := withZFS(t)
|
||||
f.on(cmd, "")
|
||||
a, err := f.machine().ScrubAct(context.Background(), act, "tank", "the operator asked")
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", act, err)
|
||||
}
|
||||
if strings.Join(a.Ran, " ") != cmd || a.After == "" || !a.Done {
|
||||
t.Errorf("%s: %+v", act, a)
|
||||
}
|
||||
if len(f.emitted) != 1 || f.emitted[0]["_type"] != ScrubEvent || f.emitted[0]["act"] != act || f.emitted[0]["why"] != "the operator asked" {
|
||||
t.Errorf("%s recorded: %+v", act, f.emitted)
|
||||
}
|
||||
}
|
||||
f := withZFS(t)
|
||||
for _, c := range []struct{ act, pool, why, says string }{
|
||||
{"start", "nope", "why", "no pool"},
|
||||
{"start", "tank", "", "say why"},
|
||||
{"start", "-s", "why", "no pool"},
|
||||
{"destroy", "tank", "why", "not an act"},
|
||||
} {
|
||||
if _, err := f.machine().ScrubAct(context.Background(), c.act, c.pool, c.why); err == nil || !strings.Contains(err.Error(), c.says) {
|
||||
t.Errorf("%+v: %v", c, err)
|
||||
}
|
||||
}
|
||||
if n := len(ran(f, "sudo")); n != 0 {
|
||||
t.Errorf("a refusal ran %v", f.calls)
|
||||
}
|
||||
f.answers["sudo -n zpool scrub -- tank"] = Ran{Code: -1, TimedOut: true}
|
||||
if _, err := f.machine().ScrubAct(context.Background(), "start", "tank", "why"); err == nil || !strings.Contains(err.Error(), "outcome unknown; check zfs_scrub") {
|
||||
t.Errorf("timed out: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -187,13 +187,13 @@
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "scrub-run",
|
||||
"id": "run",
|
||||
"type": "process",
|
||||
"name": "zfs-mesh-scrub",
|
||||
"name": "zfs-mesh-run",
|
||||
"artifact": "tools",
|
||||
"run": [
|
||||
"./zfs-tools",
|
||||
"scrub-run"
|
||||
"run"
|
||||
],
|
||||
"schedule": "0,15,30,45 * * * *"
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user