The safety review of the first version found it safe for the pool's data and asked for these fixes before it goes on the home server: - zfs_snapshot: no recursive. Every answer gives the dataset's free space and the pool's capacity, and says that the snapshot holds every block deleted or overwritten until it is destroyed by hand. - A command that runs out of time is ended with its whole process group. An act that timed out is answered from the list, or as "outcome unknown", never as "NOT DONE". - The watch runs as the operator account. - The event follower reads ZFS's queue from its start, with a cursor per boot, so the pool's import and any downtime reach the journal once. It writes "dropped N events", kills zpool before waiting for it, and exits non-zero after five failed follows in a row. - Past error counts are a warning, and counts that rise are unhealthy. - The guard test catches .Run(, exec.Command(, exec.CommandContext( and process starts. - The fixtures and the parser follow OpenZFS 2.4.4's own printers: human scan sizes under -p, no iostat section lines, error scrubs, per-vdev rebuild resilvers, paused scrubs. The operator also asked the module to own the scrub. A timer and a service of the module's own run zpool scrub -w -- <pool> from the scrub-pool, scrub-cadence, scrub-day and scrub-time settings. In the same declaration, the distribution's weekly and monthly timers for that pool are held stopped and disabled, so exactly one schedule scrubs it. zfs_scrub shows the schedule and the next run, with any resilver's progress. zfs_scrub_start, zfs_scrub_pause and zfs_scrub_stop act with why. The health check warns of a late or failed scrub and of a running resilver, and is unhealthy for a resilver that finished with errors.
111 lines
4.4 KiB
Go
111 lines
4.4 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestOnlinePoolsWithoutErrorsAreHealthyAndAnOldScrubIsOnlyAWarning(t *testing.T) {
|
|
h := withZFS(t).machine().Health(context.Background())
|
|
if !h.Healthy || len(h.Found) != 0 {
|
|
t.Fatalf("health: %+v", h)
|
|
}
|
|
if len(h.Warnings) != 1 || !strings.Contains(h.Warnings[0], "Pool rpool was last scrubbed 89 days ago, more than the module's line for a pool without a schedule (35 days)") {
|
|
t.Errorf("rpool's July scrub is a warning; tank's September one is not: %v", h.Warnings)
|
|
}
|
|
if !strings.Contains(h.Why, "tank ONLINE, 60 % full") {
|
|
t.Errorf("why: %s", h.Why)
|
|
}
|
|
}
|
|
|
|
func TestADegradedFullPoolIsUnhealthyWithEveryReason(t *testing.T) {
|
|
f := newFake(t)
|
|
f.on(line("zpool", PoolsArgv()), fixture(t, "zpool-list-full.txt"))
|
|
f.on(line("zpool", StatusArgv()), fixture(t, "zpool-status-degraded.txt"))
|
|
h := f.machine().Health(context.Background())
|
|
if h.Healthy {
|
|
t.Fatalf("healthy: %+v", h)
|
|
}
|
|
for _, want := range []string{
|
|
"Pool tank is DEGRADED: One or more devices is currently being resilvered",
|
|
"Action: Wait for the resilver to complete",
|
|
"Pool tank is 91 % full, past 90 %",
|
|
"device ata-WDC_WD120EFBX-68B0EN0_D7JA0002 in pool tank is FAULTED",
|
|
"Pool tank: 2 data errors",
|
|
} {
|
|
if !strings.Contains(h.Why, want) {
|
|
t.Errorf("why lacks %q: %s", want, h.Why)
|
|
}
|
|
}
|
|
if !strings.Contains(strings.Join(h.Warnings, " "), "ata-WDC_WD120EFBX-68B0EN0_D7JA0002 in pool tank has 12 past read errors and 107 past write errors and 3 past checksum errors") {
|
|
t.Errorf("its counts on a first look are past: %v", h.Warnings)
|
|
}
|
|
if strings.Contains(h.Why, "D7JA0009 in pool tank is") {
|
|
t.Errorf("a spare in use is not a finding: %s", h.Why)
|
|
}
|
|
}
|
|
|
|
func TestCapacityPastEightyIsAWarningAndPastNinetyUnhealthy(t *testing.T) {
|
|
st, _ := ParseStatus(fixture(t, "zpool-status-scrubbing.txt"))
|
|
now := newFake(t).now
|
|
p := Pool{Name: "tank", Capacity: 85, Health: "ONLINE"}
|
|
if h := NewErrorMemory().Judge(Look{Pools: []Pool{p}, Status: st, Now: now}); !h.Healthy || len(h.Warnings) != 1 || !strings.Contains(h.Warnings[0], "85 % full, past 80 %") {
|
|
t.Errorf("85 %%: %+v", h)
|
|
}
|
|
p.Capacity = 80
|
|
if h := NewErrorMemory().Judge(Look{Pools: []Pool{p}, Status: st, Now: now}); !h.Healthy || len(h.Warnings) != 0 {
|
|
t.Errorf("80 %% is not past 80: %+v", h)
|
|
}
|
|
p.Capacity = 92
|
|
if h := NewErrorMemory().Judge(Look{Pools: []Pool{p}, Status: st, Now: now}); h.Healthy {
|
|
t.Errorf("92 %%: %+v", h)
|
|
}
|
|
}
|
|
|
|
func TestFaultedAndNoPoolAreUnhealthy(t *testing.T) {
|
|
now := newFake(t).now
|
|
if h := NewErrorMemory().Judge(Look{Now: now}); h.Healthy || !strings.Contains(h.Why, "no pool is imported") {
|
|
t.Errorf("no pool: %+v", h)
|
|
}
|
|
st := []PoolStatus{{Name: "tank", State: "FAULTED", Status: "One or more devices could not be opened.", Errors: "No known data errors",
|
|
Root: &Device{Name: "tank", Kind: "pool", State: "FAULTED"}}}
|
|
if h := NewErrorMemory().Judge(Look{Pools: []Pool{{Name: "tank", Health: "FAULTED"}}, Status: st, Now: now}); h.Healthy || !strings.Contains(h.Why, "Pool tank is FAULTED: One or more devices could not be opened") {
|
|
t.Errorf("faulted: %+v", h)
|
|
}
|
|
}
|
|
|
|
func TestTheWatchSaysOnlyChanges(t *testing.T) {
|
|
f := withZFS(t)
|
|
var out bytes.Buffer
|
|
w := &Watch{M: f.machine(), Out: &out}
|
|
w.Look(context.Background())
|
|
w.Look(context.Background())
|
|
if n := strings.Count(out.String(), "\n"); n != 1 || !strings.Contains(out.String(), "healthy: Every pool is online with no new errors") {
|
|
t.Errorf("one line for one state: %q", out.String())
|
|
}
|
|
f.on(line("zpool", StatusArgv()), fixture(t, "zpool-status-degraded.txt"))
|
|
w.Look(context.Background())
|
|
if !strings.Contains(out.String(), "UNHEALTHY: ") {
|
|
t.Errorf("the change is said: %q", out.String())
|
|
}
|
|
}
|
|
|
|
func TestTheFollowerArgvAndAResilverStartAndFinishAreJournaled(t *testing.T) {
|
|
if strings.Join(FollowArgv(), " ") != "events -f -v" {
|
|
t.Errorf("argv: %v", FollowArgv())
|
|
}
|
|
for _, c := range []string{"sysevent.fs.zfs.resilver_start", "sysevent.fs.zfs.resilver_finish", "sysevent.fs.zfs.scrub_start",
|
|
"sysevent.fs.zfs.scrub_finish", "resource.fs.zfs.statechange", "ereport.fs.zfs.checksum"} {
|
|
if !Notable(c) {
|
|
t.Errorf("%s is not journaled", c)
|
|
}
|
|
}
|
|
var out bytes.Buffer
|
|
Follow(strings.NewReader(fixture(t, "zpool-events.txt")), &out, &Cursor{})
|
|
if !strings.Contains(out.String(), "sysevent.fs.zfs.resilver_start eid=0x14 pool=tank resilver_type=healing") {
|
|
t.Errorf("the resilver's start: %q", out.String())
|
|
}
|
|
}
|