270 lines
12 KiB
Go
270 lines
12 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func scrubConf(pool, cadence, day, at string) string {
|
|
return `{"pool":"` + pool + `","scrub-cadence":"` + cadence + `","scrub-day":"` + day + `","scrub-time":"` + at + `"}`
|
|
}
|
|
|
|
func TestTheSlotIsTheLastScheduledTimeNotAfterNow(t *testing.T) {
|
|
loc := time.Local
|
|
at := func(y int, m time.Month, d, h, min int) time.Time { return time.Date(y, m, d, h, min, 0, 0, loc) }
|
|
cases := []struct {
|
|
conf string
|
|
now time.Time
|
|
want time.Time
|
|
}{
|
|
{scrubConf("tank", "weekly", "Mon", "00:00"), at(2026, 10, 9, 12, 0), at(2026, 10, 5, 0, 0)},
|
|
{scrubConf("tank", "weekly", "Fri", "13:30"), at(2026, 10, 9, 12, 0), at(2026, 10, 2, 13, 30)},
|
|
{scrubConf("tank", "weekly", "Fri", "11:30"), at(2026, 10, 9, 12, 0), at(2026, 10, 9, 11, 30)},
|
|
{scrubConf("tank", "monthly", "Mon", "00:00"), at(2026, 10, 9, 12, 0), at(2026, 10, 5, 0, 0)},
|
|
{scrubConf("tank", "monthly", "Mon", "00:00"), at(2026, 10, 4, 23, 0), at(2026, 9, 7, 0, 0)},
|
|
{scrubConf("tank", "monthly", "Thu", "06:00"), at(2026, 10, 1, 6, 0), at(2026, 10, 1, 6, 0)},
|
|
}
|
|
for _, c := range cases {
|
|
cfg, ref := ParseConfig([]byte(c.conf), 16000)
|
|
if len(ref) != 0 {
|
|
t.Fatal(ref)
|
|
}
|
|
s := ScheduleOf(cfg)
|
|
if got := s.Slot(c.now); !got.Equal(c.want) {
|
|
t.Errorf("%q at %s: slot %s, want %s", c.conf, c.now, got, c.want)
|
|
}
|
|
if next := s.NextSlot(c.now); !next.After(c.now) || !s.Slot(next).Equal(next) {
|
|
t.Errorf("%q at %s: next slot %s", c.conf, c.now, next)
|
|
}
|
|
}
|
|
// A schedule with a value unset is no schedule; a value of another form is refused.
|
|
if cfg, _ := ParseConfig([]byte(`{"pool":"tank","scrub-cadence":"weekly"}`), 16000); ScheduleOf(cfg) != nil {
|
|
t.Error("a schedule without day and time")
|
|
}
|
|
for _, conf := range []string{scrubConf("tank", "daily", "Mon", "00:00"), scrubConf("tank", "weekly", "Monday", "00:00"),
|
|
scrubConf("tank", "weekly", "Mon", "25:00"), scrubConf("tank x", "weekly", "Mon", "00:00"), scrubConf("tank", "weekly", "Mon..Fri", "00:00"),
|
|
scrubConf("tank", "weekly", "Mon", "*:*")} {
|
|
if _, ref := ParseConfig([]byte(conf), 16000); len(ref) == 0 {
|
|
t.Errorf("accepted %q", conf)
|
|
}
|
|
}
|
|
}
|
|
|
|
// runFake is a root machine on the healthy fixtures, with the config file, the schedule state and the lock.
|
|
func runFake(t *testing.T, conf string) (*fake, *Machine, memStates, string) {
|
|
f := withZFS(t)
|
|
f.uid = 0
|
|
f.files[ConfigFile] = conf
|
|
f.files[MemInfoFile] = "MemTotal: 16384000 kB\n"
|
|
m := f.machine()
|
|
st := memStates{}
|
|
m.States = st
|
|
return f, m, st, filepath.Join(t.TempDir(), "zfs-scrub", "scrub-served.json")
|
|
}
|
|
|
|
// §20, §24, §31: the lock held: nothing run; a scrub or resilver running or an unimported pool: nothing run
|
|
// and said; a due scrub started and the run ends without waiting; a due trim by trim-day and trim-time; a
|
|
// served slot not run again; a hand act's key acted on under the lock and deleted.
|
|
func TestTheScheduledRunDoesWhatIsDueAndEndsWithoutWaiting(t *testing.T) {
|
|
const trim = `{"pool":"tank","trim-schedule":"weekly","trim-day":"Mon","trim-time":"00:00"}`
|
|
cases := []struct {
|
|
name, conf string
|
|
setup func(f *fake, st memStates)
|
|
ran, not []string
|
|
event string // event kind and reason
|
|
served bool
|
|
said string
|
|
wantErr bool
|
|
}{
|
|
{name: "scrub due", conf: scrubConf("tank", "monthly", "Mon", "00:00"), setup: func(f *fake, _ memStates) { f.on("zpool scrub -- tank", "") },
|
|
ran: []string{"zpool scrub -- tank"}, not: []string{"zpool wait"}, served: true, said: "started the scrub"},
|
|
{name: "scrub refused as one just began", conf: scrubConf("tank", "monthly", "Mon", "00:00"), setup: func(f *fake, _ memStates) {
|
|
f.answers["zpool scrub -- tank"] = Ran{Code: 1, Err: "cannot scrub tank: currently scrubbing"}
|
|
}, ran: []string{"zpool scrub -- tank"}, not: []string{"zpool wait", "scrub -p", "scrub -s"}, served: true},
|
|
{name: "scrub running", conf: scrubConf("tank", "monthly", "Mon", "00:00"), setup: func(f *fake, _ memStates) {
|
|
f.on(line("zpool", StatusArgv()), fixture(t, "zpool-status-scrubbing.txt"))
|
|
}, not: []string{"zpool scrub", "zpool wait"}, event: "run.skipped/" + SkipScanRunning},
|
|
{name: "resilver running", conf: scrubConf("tank", "monthly", "Mon", "00:00"), setup: func(f *fake, _ memStates) {
|
|
f.on(line("zpool", StatusArgv()), fixture(t, "zpool-status-degraded.txt"))
|
|
}, not: []string{"zpool scrub"}, event: "run.skipped/" + SkipScanRunning},
|
|
{name: "pool not imported", conf: scrubConf("nopool", "monthly", "Mon", "00:00"), not: []string{"zpool scrub"},
|
|
event: "run.skipped/" + SkipPoolNotImported},
|
|
{name: "slot already served by a scrub that started in it", conf: scrubConf("tank", "weekly", "Sun", "00:00"), setup: func(f *fake, _ memStates) {
|
|
f.now = time.Date(2026, 9, 15, 12, 0, 0, 0, time.Local)
|
|
}, not: []string{"zpool scrub"}, served: true, said: "ran already"},
|
|
{name: "trim due", conf: trim, setup: func(f *fake, _ memStates) { f.on("zpool trim -- tank", "") },
|
|
ran: []string{"zpool trim -- tank"}, served: true, said: "started the trim"},
|
|
{name: "trim none", conf: `{"pool":"tank","trim-schedule":"none","trim-day":"Mon","trim-time":"00:00"}`, not: []string{"zpool trim"}},
|
|
{name: "a hand act's key is acted on and deleted", conf: `{"pool":"tank"}`, setup: func(f *fake, st memStates) {
|
|
st.Put(StateSchedule, "apply", map[string]string{"by": "x"})
|
|
}, said: "apply: applied"},
|
|
{name: "a refused config runs nothing", conf: `{"pool":"tank","scrub-cadence":"hourly"}`, not: []string{"zpool"}, said: "refused"},
|
|
{name: "a failed start fails the run", conf: scrubConf("tank", "monthly", "Mon", "00:00"), setup: func(f *fake, _ memStates) {
|
|
f.answers["zpool scrub -- tank"] = Ran{Code: 1, Err: "cannot scrub: no such pool"}
|
|
}, ran: []string{"zpool scrub -- tank"}, wantErr: true},
|
|
}
|
|
for _, c := range cases {
|
|
f, m, st, served := runFake(t, c.conf)
|
|
if c.setup != nil {
|
|
c.setup(f, st)
|
|
}
|
|
said, err := m.RunScheduled(context.Background(), served)
|
|
if (err != nil) != c.wantErr {
|
|
t.Errorf("%s: error %v", c.name, err)
|
|
}
|
|
for _, r := range c.ran {
|
|
if len(ran(f, r)) != 1 {
|
|
t.Errorf("%s: %q not run once; ran %v", c.name, r, f.calls)
|
|
}
|
|
}
|
|
for _, n := range c.not {
|
|
if len(ran(f, n)) != 0 {
|
|
t.Errorf("%s: ran %v", c.name, ran(f, n))
|
|
}
|
|
}
|
|
if c.event != "" {
|
|
kind, reason, _ := strings.Cut(c.event, "/")
|
|
if len(f.emitted) != 1 || f.emitted[0]["_type"] != kind || f.emitted[0]["reason"] != reason {
|
|
t.Errorf("%s: events %v, want %s", c.name, f.emitted, c.event)
|
|
}
|
|
}
|
|
if _, err := os.Stat(served); (err == nil) != c.served {
|
|
t.Errorf("%s: served file present %v, want %v", c.name, err == nil, c.served)
|
|
}
|
|
if !strings.Contains(said, c.said) {
|
|
t.Errorf("%s: said %q, want %q", c.name, said, c.said)
|
|
}
|
|
if _, ok := st["schedule/apply"]; ok {
|
|
t.Errorf("%s: the hand act's key stayed", c.name)
|
|
}
|
|
}
|
|
// The lock held by another: nothing run, said.
|
|
f, m, _, served := runFake(t, scrubConf("tank", "monthly", "Mon", "00:00"))
|
|
m.Lock = func(string, bool) (func(), bool, error) { return nil, false, nil }
|
|
if said, err := m.RunScheduled(context.Background(), served); err != nil || !strings.Contains(said, SkipLockHeld) ||
|
|
len(f.emitted) != 1 || f.emitted[0]["reason"] != SkipLockHeld || len(ran(f, "zpool")) != 0 {
|
|
t.Errorf("lock held: %q %v %v %v", said, err, f.emitted, f.calls)
|
|
}
|
|
}
|
|
|
|
// scheduled is the healthy fixtures with the module's scrub schedule for tank, its timer waiting and its
|
|
// last run a success.
|
|
func scheduled(t *testing.T) *fake {
|
|
f := withZFS(t)
|
|
f.files[ConfigFile] = scrubConf("tank", "weekly", "Mon", "00:00")
|
|
f.files[MemInfoFile] = "MemTotal: 16384000 kB\n"
|
|
f.on(line("systemctl", TimerShowArgv(RunTimer)),
|
|
"ActiveState=active\nUnitFileState=enabled\nNextElapseUSecRealtime=@1791842400\nLastTriggerUSec=@1791237600\n")
|
|
f.on(line("systemctl", ServiceShowArgv(RunService)),
|
|
"ActiveState=inactive\nResult=success\nExecMainStartTimestamp=@1791237600\nExecMainExitTimestamp=@1791331200\n")
|
|
return f
|
|
}
|
|
|
|
func TestTheScheduleIsTheConfigAndWhatTheTimerSays(t *testing.T) {
|
|
f := scheduled(t)
|
|
got, err := f.machine().Scrubs(context.Background(), false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
a := got.(ScrubAnswer)
|
|
s := a.Schedule
|
|
if s == nil || s.Pool != "tank" || s.Cadence != "weekly" || s.Timer != "zfs-mesh-run.timer" || s.TimerState != "active" ||
|
|
s.Enabled != "enabled" || s.Next == nil || !s.Next.Equal(time.Date(2026, 10, 12, 0, 0, 0, 0, time.Local)) || s.LastResult != "success" {
|
|
t.Fatalf("schedule: %+v", s)
|
|
}
|
|
f = withZFS(t)
|
|
got, _ = f.machine().Scrubs(context.Background(), false)
|
|
if a := got.(ScrubAnswer); a.Schedule != nil || !strings.Contains(strings.Join(a.Said, " "), "no scrub schedule") {
|
|
t.Errorf("no config: %+v", a)
|
|
}
|
|
}
|
|
|
|
func TestTheScheduleWarnsOfAnOldScrubAFailedRunAndAStoppedTimer(t *testing.T) {
|
|
f := scheduled(t)
|
|
h := f.machine().Health(context.Background())
|
|
if !h.Healthy {
|
|
t.Fatalf("%+v", h)
|
|
}
|
|
w := strings.Join(h.Warnings, " ")
|
|
// tank's last scrub finished 2026-09-14, 25 days before the fake clock: past weekly plus its margin.
|
|
if !strings.Contains(w, "Pool tank was last scrubbed 25 days ago, more than its weekly schedule allows (9 days)") ||
|
|
!strings.Contains(w, "The scheduled scrub of tank due 2026-10-05 00:00 has not started") {
|
|
t.Errorf("warnings: %v", h.Warnings)
|
|
}
|
|
f.on(line("systemctl", ServiceShowArgv(RunService)),
|
|
"ActiveState=failed\nResult=exit-code\nExecMainStartTimestamp=@1791237600\nExecMainExitTimestamp=@1791237601\n")
|
|
f.on(line("systemctl", TimerShowArgv(RunTimer)),
|
|
"ActiveState=inactive\nUnitFileState=disabled\nNextElapseUSecRealtime=\nLastTriggerUSec=@1791237600\n")
|
|
w = strings.Join(f.machine().Health(context.Background()).Warnings, " ")
|
|
for _, want := range []string{"The last scheduled scrub of tank failed (exit-code)", "The scrub schedule's timer zfs-mesh-run.timer is inactive and disabled"} {
|
|
if !strings.Contains(w, want) {
|
|
t.Errorf("lacks %q: %s", want, w)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestTheScrubActsAreFixedCommandsWithWhyReadBackAndRecorded(t *testing.T) {
|
|
for act, cmd := range map[string]string{
|
|
"start": "sudo -n zpool scrub -- tank",
|
|
"pause": "sudo -n zpool scrub -p -- tank",
|
|
"stop": "sudo -n zpool scrub -s -- tank",
|
|
} {
|
|
f := withZFS(t)
|
|
f.on(cmd, "")
|
|
a, err := f.machine().ScrubAct(context.Background(), act, "tank", "the operator asked")
|
|
if err != nil {
|
|
t.Fatalf("%s: %v", act, err)
|
|
}
|
|
if strings.Join(a.Ran, " ") != cmd || a.After == "" || !a.Done {
|
|
t.Errorf("%s: %+v", act, a)
|
|
}
|
|
if len(f.emitted) != 1 || f.emitted[0]["_type"] != ScrubEvent || f.emitted[0]["act"] != act || f.emitted[0]["why"] != "the operator asked" {
|
|
t.Errorf("%s recorded: %+v", act, f.emitted)
|
|
}
|
|
}
|
|
f := withZFS(t)
|
|
for _, c := range []struct{ act, pool, why, says string }{
|
|
{"start", "nope", "why", "no pool"},
|
|
{"start", "tank", "", "say why"},
|
|
{"start", "-s", "why", "no pool"},
|
|
{"destroy", "tank", "why", "not an act"},
|
|
} {
|
|
if _, err := f.machine().ScrubAct(context.Background(), c.act, c.pool, c.why); err == nil || !strings.Contains(err.Error(), c.says) {
|
|
t.Errorf("%+v: %v", c, err)
|
|
}
|
|
}
|
|
if n := len(ran(f, "sudo")); n != 0 {
|
|
t.Errorf("a refusal ran %v", f.calls)
|
|
}
|
|
f.answers["sudo -n zpool scrub -- tank"] = Ran{Code: -1, TimedOut: true}
|
|
if _, err := f.machine().ScrubAct(context.Background(), "start", "tank", "why"); err == nil || !strings.Contains(err.Error(), "outcome unknown; check zfs_scrub") {
|
|
t.Errorf("timed out: %v", err)
|
|
}
|
|
}
|
|
|
|
// Third review: the scheduled run's state is root's alone — a folder of its own, 0700 — so the operator account, or anything it runs, cannot make the run write elsewhere.
|
|
func TestTheServedSlotIsWrittenOnlyIntoAPrivateFolderAndNeverThroughALink(t *testing.T) {
|
|
dir := filepath.Join(t.TempDir(), "zfs-scrub")
|
|
path := filepath.Join(dir, "scrub-served.json")
|
|
if err := writeServed(path, map[string]int64{"tank": 1}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if fi, err := os.Lstat(dir); err != nil || fi.Mode().Perm() != 0o700 || !fi.IsDir() {
|
|
t.Errorf("the folder: %v %v", fi.Mode(), err)
|
|
}
|
|
// A folder open to others: refused. (The refusal of a link is not run here: no test of this module makes a
|
|
// symbolic link, the mesh's hard rule.)
|
|
open := filepath.Join(t.TempDir(), "open")
|
|
os.Mkdir(open, 0o755)
|
|
if err := writeServed(filepath.Join(open, "scrub-served.json"), map[string]int64{"tank": 3}); err == nil {
|
|
t.Errorf("written into a folder others can read")
|
|
}
|
|
if filepath.Dir(ServedFile) == filepath.Dir(ErrorsFile) {
|
|
t.Errorf("the run's state shares the account's folder")
|
|
}
|
|
}
|