package main import ( "context" "fmt" "strings" "time" ) // The lines a pool's capacity is judged against: past CapacityWarn a warning, past CapacityFail // unhealthy. ZFS slows as a pool fills, and a full pool can refuse even a deletion. const ( CapacityWarn = 80 CapacityFail = 90 ) // Health is the module's health, as the node-engine asks it (novox/hq ADR 0240): unhealthy with why, one // sentence per finding; warnings do not make it unhealthy. type Health struct { Healthy bool `json:"healthy"` Why string `json:"why"` Found []string `json:"found,omitempty"` Warnings []string `json:"warnings,omitempty"` } // ErrorsFresh is how long a rise in a device's error counts keeps the health unhealthy: long enough for // the node-engine's two looks to see it, and a condition to be raised. const ErrorsFresh = 15 * time.Minute // Look is what one health check read. type Look struct { Pools []Pool Status []PoolStatus Now time.Time Schedule *Schedule // ScheduleErr is a schedule whose settings are not valid. ScheduleErr error // Served is the last slot the scheduled run served, per pool. Served map[string]int64 } // errorFinding judges one line of status's error counts against the last look: a count that rose is a // fault now; a count that was there before is past, a warning to clear by hand once understood. func (mem *ErrorMemory) errorFinding(pool string, d *Device, now time.Time) (fault, past string) { key := pool + "/" + d.Kind + "/" + d.Name counts := [3]uint64{d.Read, d.Write, d.Checksum} names := [3]string{"read", "write", "checksum"} if mem == nil { mem = NewErrorMemory() } mem.mu.Lock() defer mem.mu.Unlock() before, seen := mem.counts[key] mem.counts[key] = counts var rose []string for i := range counts { if seen && counts[i] > before[i] { rose = append(rose, fmt.Sprintf("%s errors rose from %d to %d", names[i], before[i], counts[i])) } } if len(rose) > 0 { mem.rose[key] = now return fmt.Sprintf("%s %s in pool %s: %s since the last look.", d.Kind, d.Name, pool, strings.Join(rose, ", ")), "" } if at, ok := mem.rose[key]; ok && now.Sub(at) < ErrorsFresh { return fmt.Sprintf("%s %s in pool %s: its errors rose at %s (read %d, write %d, checksum %d).", d.Kind, d.Name, pool, at.Local().Format("15:04"), d.Read, d.Write, d.Checksum), "" } if d.Errors() == 0 { return "", "" } var parts []string for i := range counts { if counts[i] > 0 { parts = append(parts, fmt.Sprintf("%d past %s errors", counts[i], names[i])) } } return "", fmt.Sprintf("%s %s in pool %s has %s; clear them by hand once understood.", d.Kind, d.Name, pool, strings.Join(parts, " and ")) } // Judge judges the pools from what one look read. Unhealthy is a fault now: a pool or device not online, // error counts that rose, data errors, a pool past CapacityFail, a resilver that finished with errors. // A warning is a past fault or a risk: error counts that were there before, a pool past CapacityWarn, a // resilver running, a scrub later than its schedule allows, a schedule that failed or does not run. func (mem *ErrorMemory) Judge(l Look) Health { var found, warn []string if len(l.Pools) == 0 { found = append(found, "ZFS is on this machine and no pool is imported.") } if l.ScheduleErr != nil { warn = append(warn, sentence("the scrub schedule's settings are not valid: "+l.ScheduleErr.Error())) } for _, p := range l.Pools { s := statusOf(l.Status, p.Name) state := p.Health if s != nil && s.State != "" { state = s.State } if state != "ONLINE" { line := fmt.Sprintf("Pool %s is %s", p.Name, state) if s != nil && s.Status != "" { line += ": " + strings.TrimSuffix(s.Status, ".") } if s != nil && s.Action != "" { line += ". Action: " + strings.TrimSuffix(s.Action, ".") } found = append(found, line+".") } switch { case p.Capacity > CapacityFail: found = append(found, fmt.Sprintf("Pool %s is %d %% full, past %d %%: %s free.", p.Name, p.Capacity, CapacityFail, bytesSaid(p.Free))) case p.Capacity > CapacityWarn: warn = append(warn, fmt.Sprintf("Pool %s is %d %% full, past %d %%: %s free.", p.Name, p.Capacity, CapacityWarn, bytesSaid(p.Free))) } if s == nil { found = append(found, fmt.Sprintf("zpool status said nothing of pool %s.", p.Name)) continue } walk(s.Root, func(d *Device, _ int) { if d.Kind == "section" { return } fault, past := mem.errorFinding(p.Name, d, l.Now) if fault != "" { found = append(found, sentence(fault)) } if past != "" { warn = append(warn, sentence(past)) } if d.Kind != "pool" && d.State != "" && d.State != "ONLINE" && d.State != "AVAIL" && d.State != "INUSE" { // A device not online already makes its pool DEGRADED or worse; said here by name. found = append(found, fmt.Sprintf("%s %s in pool %s is %s.", d.Kind, d.Name, p.Name, d.State)) } }, 0) if e := strings.TrimSpace(s.Errors); e != "" && e != "No known data errors" { found = append(found, fmt.Sprintf("Pool %s: %s.", p.Name, strings.TrimSuffix(e, "."))) } for _, r := range s.Resilvers() { what := "the resilver" if r.Vdev != "" { what += " of " + r.Vdev } switch { case r.State == "running": line := fmt.Sprintf("A resilver") if r.Vdev != "" { line += " of " + r.Vdev } line += fmt.Sprintf(" in pool %s is running", p.Name) if r.Done != nil { line += fmt.Sprintf(": %.2f %% done", *r.Done) } if r.ToGo != "" { line += ", " + r.ToGo + " to go" } warn = append(warn, sentence(strings.ToLower(line[:1])+line[1:])) case r.State == "finished" && deref(r.Errors) > 0: found = append(found, sentence(fmt.Sprintf("%s in pool %s finished with %d errors, %s", what, p.Name, deref(r.Errors), stamp(r.Ended)))) } } late, by := ScrubLate(p.Name, l.Schedule) ended := LastScrubEnded(*s) switch { case ended != nil && l.Now.Sub(*ended) > late: warn = append(warn, fmt.Sprintf("Pool %s was last scrubbed %.0f days ago, more than %s (%d days).", p.Name, l.Now.Sub(*ended).Hours()/24, by, int(late.Hours()/24))) case len(s.Scans) == 0: warn = append(warn, fmt.Sprintf("Pool %s was never scrubbed.", p.Name)) } if l.Schedule != nil && l.Schedule.Pool == p.Name { warn = append(warn, scheduleFindings(l.Schedule, s, l.Now, l.Served)...) } } if l.Schedule != nil && statusOf(l.Status, l.Schedule.Pool) == nil { warn = append(warn, fmt.Sprintf("The scrub schedule names pool %s, which is not imported here.", l.Schedule.Pool)) } mem.save() if len(found) > 0 { return Health{Healthy: false, Why: strings.Join(found, " "), Found: found, Warnings: warn} } var parts []string for _, p := range l.Pools { parts = append(parts, fmt.Sprintf("%s ONLINE, %d %% full", p.Name, p.Capacity)) } why := "Every pool is online with no new errors: " + strings.Join(parts, "; ") + "." if len(warn) > 0 { why += " Warnings: " + strings.Join(warn, " ") } return Health{Healthy: true, Why: why, Warnings: warn} } // Health reads the pools and judges them. A machine without ZFS is unhealthy, loudly: the module was put // where it does not apply. func (m *Machine) Health(ctx context.Context) Health { pools, err := m.Pools(ctx) if err != nil { if a, ok := err.(errAbsent); ok { return Health{Healthy: false, Why: sentence(a.why) + " Unassign the module from this machine.", Found: []string{a.why}} } return Health{Healthy: false, Why: sentence("zpool list did not answer: " + err.Error()), Found: []string{err.Error()}} } status, err := m.Status(ctx) if err != nil { return Health{Healthy: false, Why: sentence("zpool status did not answer: " + err.Error()), Found: []string{err.Error()}} } sched, serr := m.ScheduleNow(ctx) return m.Errors.Judge(Look{Pools: pools, Status: status, Now: m.Now(), Schedule: sched, ScheduleErr: serr, Served: m.readServed(ServedFile)}) } func sentence(s string) string { s = strings.TrimSpace(s) if s == "" { return s } s = strings.ToUpper(s[:1]) + s[1:] if !strings.HasSuffix(s, ".") { s += "." } return s }