package main import ( "context" "encoding/json" "fmt" "strings" "time" ) // What the module's own work adds to its health (design 42 zfs-owns-zfs §29, §15). // // Unhealthy: a `refused` (the hostid's among them) or a failed command in the state `applied`, naming the key and // why. Warnings: a wait, naming on what and since when; a parameter awaiting a load of the kernel module; a // `.pacnew` beside a package file; the last scheduled ZFS snapshot of a kind older than its cadence; `autoreplace` on // or a spare in the pool while `zed` is not `on`; the pool above 80 % with `snapshots` set; scheduled ZFS snapshots // kept after `snapshots` went to `none` (or a kind's keep count to 0); `paused` set. // ModuleLook is what one health check read of the module's own config, states and ledger. type ModuleLook struct { Config *Config Applied map[string]AppliedEntry Paused bool // Ledger is the scheduled ZFS snapshots; LedgerKnown says whether it could be read. Ledger []LedgerEntry LedgerKnown bool // Pacnew are the package files a `.pacnew` lies beside. Pacnew []string } // snapshotLate is how old the last scheduled ZFS snapshot of a kind may be before it is a warning: its cadence and // the margin for a run that waited for the lock or a full pool. var snapshotLate = map[string]time.Duration{"hourly": 2 * time.Hour, "daily": 36 * time.Hour, "weekly": 9 * 24 * time.Hour, "monthly": 38 * 24 * time.Hour} // moduleLook reads what the module itself keeps. It answers nil when the config file is not readable or not valid: // the module is then judged on its pools alone, and the refusal is already in the state `applied`. func (m *Machine) moduleLook(ctx context.Context) *ModuleLook { text, err := m.ReadFile(ConfigFile) if err != nil { return nil } c, refused := ParseConfig(text, m.memMiB()) if len(refused) > 0 { c = nil } l := &ModuleLook{Config: c, Applied: map[string]AppliedEntry{}} if m.States != nil { if keys, err := m.States.Keys(StateApplied); err == nil { for _, k := range keys { if raw, ok, _ := m.States.Get(StateApplied, k); ok { var e AppliedEntry if json.Unmarshal(raw, &e) == nil { l.Applied[k] = e } } } } } l.Paused = m.Paused() if m.Ledger != nil { if es, err := m.ledgerAll(ctx); err == nil { l.Ledger, l.LedgerKnown = es, true } } for _, p := range []string{"/etc/default/zfs", "/etc/zfs/zed.d/zed.rc"} { if m.exists(p + ".pacnew") { l.Pacnew = append(l.Pacnew, p) } } return l } // moduleFindings judges the module's own work. func (l Look) moduleFindings() (found, warn []string) { k := l.Module for _, key := range sortedKeys(k.Applied) { e := k.Applied[key] switch e.Outcome { case OutcomeRefused: found = append(found, sentence(fmt.Sprintf("%s is refused: %s", key, e.Why))) case OutcomeFailed: found = append(found, sentence(fmt.Sprintf("a command for %s failed: %s", key, e.Why))) case OutcomeWaiting: since := "a time not said" if !e.Since.IsZero() { since = e.Since.Local().Format("2006-01-02 15:04") } warn = append(warn, sentence(fmt.Sprintf("%s waits on %s since %s", key, e.WaitsOn, since))) case OutcomeAwaitingLoad: warn = append(warn, sentence(fmt.Sprintf("%s takes effect at the next load of the kernel module", key))) } } for _, p := range k.Pacnew { warn = append(warn, sentence(p+".pacnew lies beside it: the package shipped a newer file the module does not take")) } if k.Paused { warn = append(warn, "The scheduled ZFS snapshots are paused: none is taken or removed.") } c := k.Config if c == nil { return found, warn } if c.Zed != "on" { spare := false for _, p := range l.Status { if p.Name == c.Pool { if s := sectionOf(&p, "spare"); s != nil && len(s.Children) > 0 { spare = true } } } if c.PoolProperties["autoreplace"] == "on" || spare { warn = append(warn, "Pool "+c.Pool+" has autoreplace on or a spare, and zed is not on: nothing replaces a failed device by itself.") } } if c.Snapshots != nil && !c.Snapshots.None { for _, p := range l.Pools { if p.Name == c.Pool && p.Capacity > FullPercent { warn = append(warn, fmt.Sprintf("Pool %s is %d %% full, past %d %%: no scheduled ZFS snapshot is taken.", p.Name, p.Capacity, FullPercent)) } } } if !k.LedgerKnown { return found, warn } if c.Snapshots != nil && !c.Snapshots.None { for _, kind := range keepsOf(c.Snapshots) { var last time.Time for _, e := range k.Ledger { if at, err := time.Parse(time.RFC3339, e.At); err == nil && e.Kind == kind && at.After(last) { last = at } } if !last.IsZero() && l.Now.Sub(last) > snapshotLate[kind] { warn = append(warn, fmt.Sprintf("The last scheduled %s ZFS snapshot is from %s, older than its cadence allows.", kind, last.Local().Format("2006-01-02 15:04"))) } } } kept := map[string][]string{} for _, e := range k.Ledger { if c.Snapshots == nil || c.Snapshots.None || c.Snapshots.Of(e.Kind) == 0 { kept[e.Kind] = append(kept[e.Kind], e.Dataset+"@"+e.Name) } } for _, kind := range keepKinds { if len(kept[kind]) > 0 { warn = append(warn, fmt.Sprintf("%d scheduled %s ZFS snapshots are kept though snapshots does not ask for them (none or a keep count of 0 removes nothing): %s.", len(kept[kind]), kind, strings.Join(kept[kind], ", "))) } } return found, warn }