182 lines
8.4 KiB
Go
182 lines
8.4 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestOnlinePoolsWithoutErrorsAreHealthyAndAnOldScrubIsOnlyAWarning(t *testing.T) {
|
|
h := withZFS(t).machine().Health(context.Background())
|
|
if !h.Healthy || len(h.Found) != 0 {
|
|
t.Fatalf("health: %+v", h)
|
|
}
|
|
if len(h.Warnings) != 1 || !strings.Contains(h.Warnings[0], "Pool rpool was last scrubbed 89 days ago, more than the module's line for a pool without a schedule (35 days)") {
|
|
t.Errorf("rpool's July scrub is a warning; tank's September one is not: %v", h.Warnings)
|
|
}
|
|
if !strings.Contains(h.Why, "tank ONLINE, 60 % full") {
|
|
t.Errorf("why: %s", h.Why)
|
|
}
|
|
}
|
|
|
|
func TestADegradedFullPoolIsUnhealthyWithEveryReason(t *testing.T) {
|
|
f := newFake(t)
|
|
f.on(line("zpool", PoolsArgv()), fixture(t, "zpool-list-full.txt"))
|
|
f.on(line("zpool", StatusArgv()), fixture(t, "zpool-status-degraded.txt"))
|
|
h := f.machine().Health(context.Background())
|
|
if h.Healthy {
|
|
t.Fatalf("healthy: %+v", h)
|
|
}
|
|
for _, want := range []string{
|
|
"Pool tank is DEGRADED: One or more devices is currently being resilvered",
|
|
"Action: Wait for the resilver to complete",
|
|
"Pool tank is 91 % full, past 90 %",
|
|
"device ata-WDC_WD120EFBX-68B0EN0_D7JA0002 in pool tank is FAULTED",
|
|
"Pool tank: 2 data errors",
|
|
} {
|
|
if !strings.Contains(h.Why, want) {
|
|
t.Errorf("why lacks %q: %s", want, h.Why)
|
|
}
|
|
}
|
|
if !strings.Contains(strings.Join(h.Warnings, " "), "ata-WDC_WD120EFBX-68B0EN0_D7JA0002 in pool tank has 12 past read errors and 107 past write errors and 3 past checksum errors") {
|
|
t.Errorf("its counts on a first look are past: %v", h.Warnings)
|
|
}
|
|
if strings.Contains(h.Why, "D7JA0009 in pool tank is") {
|
|
t.Errorf("a spare in use is not a finding: %s", h.Why)
|
|
}
|
|
}
|
|
|
|
func TestCapacityPastEightyIsAWarningAndPastNinetyUnhealthy(t *testing.T) {
|
|
st, _ := ParseStatus(fixture(t, "zpool-status-scrubbing.txt"))
|
|
now := newFake(t).now
|
|
p := Pool{Name: "tank", Capacity: 85, Health: "ONLINE"}
|
|
if h := NewErrorMemory().Judge(Look{Pools: []Pool{p}, Status: st, Now: now}); !h.Healthy || len(h.Warnings) != 1 || !strings.Contains(h.Warnings[0], "85 % full, past 80 %") {
|
|
t.Errorf("85 %%: %+v", h)
|
|
}
|
|
p.Capacity = 80
|
|
if h := NewErrorMemory().Judge(Look{Pools: []Pool{p}, Status: st, Now: now}); !h.Healthy || len(h.Warnings) != 0 {
|
|
t.Errorf("80 %% is not past 80: %+v", h)
|
|
}
|
|
p.Capacity = 92
|
|
if h := NewErrorMemory().Judge(Look{Pools: []Pool{p}, Status: st, Now: now}); h.Healthy {
|
|
t.Errorf("92 %%: %+v", h)
|
|
}
|
|
}
|
|
|
|
func TestFaultedAndNoPoolAreUnhealthy(t *testing.T) {
|
|
now := newFake(t).now
|
|
if h := NewErrorMemory().Judge(Look{Now: now}); h.Healthy || !strings.Contains(h.Why, "no pool is imported") {
|
|
t.Errorf("no pool: %+v", h)
|
|
}
|
|
st := []PoolStatus{{Name: "tank", State: "FAULTED", Status: "One or more devices could not be opened.", Errors: "No known data errors",
|
|
Root: &Device{Name: "tank", Kind: "pool", State: "FAULTED"}}}
|
|
if h := NewErrorMemory().Judge(Look{Pools: []Pool{{Name: "tank", Health: "FAULTED"}}, Status: st, Now: now}); h.Healthy || !strings.Contains(h.Why, "Pool tank is FAULTED: One or more devices could not be opened") {
|
|
t.Errorf("faulted: %+v", h)
|
|
}
|
|
}
|
|
|
|
func TestTheWatchSaysOnlyChanges(t *testing.T) {
|
|
f := withZFS(t)
|
|
var out bytes.Buffer
|
|
w := &Watch{M: f.machine(), Out: &out}
|
|
w.Look(context.Background())
|
|
w.Look(context.Background())
|
|
if n := strings.Count(out.String(), "\n"); n != 1 || !strings.Contains(out.String(), "healthy: Every pool is online with no new errors") {
|
|
t.Errorf("one line for one state: %q", out.String())
|
|
}
|
|
f.on(line("zpool", StatusArgv()), fixture(t, "zpool-status-degraded.txt"))
|
|
w.Look(context.Background())
|
|
if !strings.Contains(out.String(), "UNHEALTHY: ") {
|
|
t.Errorf("the change is said: %q", out.String())
|
|
}
|
|
}
|
|
|
|
func TestTheFollowerArgvAndAResilverStartAndFinishAreJournaled(t *testing.T) {
|
|
if strings.Join(FollowArgv(), " ") != "events -f -v" {
|
|
t.Errorf("argv: %v", FollowArgv())
|
|
}
|
|
for _, c := range []string{"sysevent.fs.zfs.resilver_start", "sysevent.fs.zfs.resilver_finish", "sysevent.fs.zfs.scrub_start",
|
|
"sysevent.fs.zfs.scrub_finish", "resource.fs.zfs.statechange", "ereport.fs.zfs.checksum"} {
|
|
if !Notable(c) {
|
|
t.Errorf("%s is not journaled", c)
|
|
}
|
|
}
|
|
var out bytes.Buffer
|
|
Follow(strings.NewReader(fixture(t, "zpool-events.txt")), &out, &Cursor{})
|
|
if !strings.Contains(out.String(), "sysevent.fs.zfs.resilver_start eid=0x14 pool=tank resilver_type=healing") {
|
|
t.Errorf("the resilver's start: %q", out.String())
|
|
}
|
|
}
|
|
|
|
// §29, §15: a refused (the hostid's among them) or failed command in the state `applied` is unhealthy naming the key
|
|
// and why, and healthy again once it is gone; each warning of §29 is a warning and leaves the module healthy.
|
|
func TestTheModulesOwnWorkAddsToItsHealthAsSection29Says(t *testing.T) {
|
|
st, _ := ParseStatus(fixture(t, "zpool-status-scrubbing.txt"))
|
|
now := newFake(t).now
|
|
snap := func(json string) *Config {
|
|
c, ref := ParseConfig([]byte(json), 16000)
|
|
if len(ref) != 0 {
|
|
t.Fatal(ref)
|
|
}
|
|
return c
|
|
}
|
|
entry := func(kind string, h int) LedgerEntry {
|
|
return LedgerEntry{"tank", "mesh-" + kind + "-x", kind, now.Add(-time.Duration(h) * time.Hour).UTC().Format(time.RFC3339)}
|
|
}
|
|
cases := []struct {
|
|
name string
|
|
look ModuleLook
|
|
capacity int
|
|
unhealthy string
|
|
warning string
|
|
}{
|
|
{name: "hostid refused", look: ModuleLook{Applied: map[string]AppliedEntry{"hostid": {Outcome: OutcomeRefused, Why: "the pool's import would need -f"}}},
|
|
unhealthy: "Hostid is refused: the pool's import would need -f"},
|
|
{name: "a failed command", look: ModuleLook{Applied: map[string]AppliedEntry{"cache-devices": {Outcome: OutcomeFailed, Why: "zpool add failed"}}},
|
|
unhealthy: "a command for cache-devices failed: zpool add failed"},
|
|
{name: "a wait", look: ModuleLook{Applied: map[string]AppliedEntry{"cache-devices": {Outcome: OutcomeWaiting, WaitsOn: "a scrub of tank", Since: now}}},
|
|
warning: "waits on a scrub of tank since"},
|
|
{name: "awaiting a load", look: ModuleLook{Applied: map[string]AppliedEntry{"module-options.zfs_txg_timeout": {Outcome: OutcomeAwaitingLoad}}},
|
|
warning: "takes effect at the next load of the kernel module"},
|
|
{name: "a pacnew", look: ModuleLook{Pacnew: []string{"/etc/default/zfs"}}, warning: "/etc/default/zfs.pacnew"},
|
|
{name: "paused", look: ModuleLook{Paused: true}, warning: "are paused"},
|
|
{name: "last snapshot older than its cadence", look: ModuleLook{Config: snap(`{"pool":"tank","snapshots":{"hourly":2}}`), LedgerKnown: true,
|
|
Ledger: []LedgerEntry{entry("hourly", 5)}}, warning: "last scheduled hourly ZFS snapshot"},
|
|
{name: "autoreplace without zed", look: ModuleLook{Config: snap(`{"pool":"tank","pool-properties":{"autoreplace":"on"}}`)}, warning: "autoreplace on or a spare"},
|
|
{name: "pool above 80 % with snapshots", look: ModuleLook{Config: snap(`{"pool":"tank","snapshots":{"daily":2}}`)}, capacity: 85, warning: "no scheduled ZFS snapshot is taken"},
|
|
{name: "kept after none", look: ModuleLook{Config: snap(`{"pool":"tank","snapshots":{"none":true}}`), LedgerKnown: true, Ledger: []LedgerEntry{entry("daily", 1)}},
|
|
warning: "kept though snapshots does not ask for them"},
|
|
{name: "kept at a keep count of 0", look: ModuleLook{Config: snap(`{"pool":"tank","snapshots":{"hourly":0,"daily":1}}`), LedgerKnown: true,
|
|
Ledger: []LedgerEntry{entry("hourly", 1), entry("daily", 1)}}, warning: "1 scheduled hourly ZFS snapshots are kept"},
|
|
{name: "nothing to say", look: ModuleLook{Config: snap(`{"pool":"tank","snapshots":{"daily":2}}`), LedgerKnown: true, Ledger: []LedgerEntry{entry("daily", 1)}}},
|
|
}
|
|
for _, c := range cases {
|
|
look := c.look
|
|
if look.Applied == nil {
|
|
look.Applied = map[string]AppliedEntry{}
|
|
}
|
|
cap := c.capacity
|
|
if cap == 0 {
|
|
cap = 50
|
|
}
|
|
h := NewErrorMemory().Judge(Look{Pools: []Pool{{Name: "tank", Capacity: cap, Health: "ONLINE"}}, Status: st, Now: now, Module: &look})
|
|
if c.unhealthy != "" {
|
|
if h.Healthy || !strings.Contains(strings.ToLower(h.Why), strings.ToLower(c.unhealthy)) {
|
|
t.Errorf("%s: %+v", c.name, h)
|
|
}
|
|
continue
|
|
}
|
|
all := strings.Join(h.Warnings, " ")
|
|
if !h.Healthy || (c.warning == "" && len(h.Warnings) != 0) || (c.warning != "" && !strings.Contains(all, c.warning)) {
|
|
t.Errorf("%s: %+v", c.name, h)
|
|
}
|
|
}
|
|
// Once the refusal is gone the module is healthy again.
|
|
if h := NewErrorMemory().Judge(Look{Pools: []Pool{{Name: "tank", Capacity: 50, Health: "ONLINE"}}, Status: st, Now: now,
|
|
Module: &ModuleLook{Applied: map[string]AppliedEntry{}}}); !h.Healthy {
|
|
t.Errorf("after the refusal: %+v", h)
|
|
}
|
|
}
|