mesh-catalog/modules/zfs/cmd/zfs-tools/health.go

package main

import (
	"context"
	"fmt"
	"strings"
	"time"
)

// The lines a pool's capacity is judged against: past CapacityWarn a warning, past CapacityFail
// unhealthy. ZFS slows as a pool fills, and a full pool can refuse even a deletion.
const (
	CapacityWarn = 80
	CapacityFail = 90
)

// Health is the module's health, as the node-engine asks it (novox/hq ADR 0240): unhealthy with why, one
// sentence per finding; warnings do not make it unhealthy.
type Health struct {
	Healthy  bool     `json:"healthy"`
	Why      string   `json:"why"`
	Found    []string `json:"found,omitempty"`
	Warnings []string `json:"warnings,omitempty"`
}

// ErrorsFresh is how long a rise in a device's error counts keeps the health unhealthy: long enough for
// the node-engine's two looks to see it, and a condition to be raised.
const ErrorsFresh = 15 * time.Minute

// Look is what one health check read.
type Look struct {
	Pools    []Pool
	Status   []PoolStatus
	Now      time.Time
	Schedule *Schedule
	// ScheduleErr is a schedule whose settings are not valid.
	ScheduleErr error
	// Served is the last slot the scheduled run served, per pool.
	Served map[string]int64
}

// errorFinding judges one line of status's error counts against the last look: a count that rose is a
// fault now; a count that was there before is past, a warning to clear by hand once understood.
func (mem *ErrorMemory) errorFinding(pool string, d *Device, now time.Time) (fault, past string) {
	key := pool + "/" + d.Kind + "/" + d.Name
	counts := [3]uint64{d.Read, d.Write, d.Checksum}
	names := [3]string{"read", "write", "checksum"}
	if mem == nil {
		mem = NewErrorMemory()
	}
	mem.mu.Lock()
	defer mem.mu.Unlock()
	before, seen := mem.counts[key]
	mem.counts[key] = counts
	var rose []string
	for i := range counts {
		if seen && counts[i] > before[i] {
			rose = append(rose, fmt.Sprintf("%s errors rose from %d to %d", names[i], before[i], counts[i]))
		}
	}
	if len(rose) > 0 {
		mem.rose[key] = now
		return fmt.Sprintf("%s %s in pool %s: %s since the last look.", d.Kind, d.Name, pool, strings.Join(rose, ", ")), ""
	}
	if at, ok := mem.rose[key]; ok && now.Sub(at) < ErrorsFresh {
		return fmt.Sprintf("%s %s in pool %s: its errors rose at %s (read %d, write %d, checksum %d).", d.Kind, d.Name, pool,
			at.Local().Format("15:04"), d.Read, d.Write, d.Checksum), ""
	}
	if d.Errors() == 0 {
		return "", ""
	}
	var parts []string
	for i := range counts {
		if counts[i] > 0 {
			parts = append(parts, fmt.Sprintf("%d past %s errors", counts[i], names[i]))
		}
	}
	return "", fmt.Sprintf("%s %s in pool %s has %s; clear them by hand once understood.", d.Kind, d.Name, pool, strings.Join(parts, " and "))
}

// Judge judges the pools from what one look read. Unhealthy is a fault now: a pool or device not online,
// error counts that rose, data errors, a pool past CapacityFail, a resilver that finished with errors.
// A warning is a past fault or a risk: error counts that were there before, a pool past CapacityWarn, a
// resilver running, a scrub later than its schedule allows, a schedule that failed or does not run.
func (mem *ErrorMemory) Judge(l Look) Health {
	var found, warn []string
	if len(l.Pools) == 0 {
		found = append(found, "ZFS is on this machine and no pool is imported.")
	}
	if l.ScheduleErr != nil {
		warn = append(warn, sentence("the scrub schedule's settings are not valid: "+l.ScheduleErr.Error()))
	}
	for _, p := range l.Pools {
		s := statusOf(l.Status, p.Name)
		state := p.Health
		if s != nil && s.State != "" {
			state = s.State
		}
		if state != "ONLINE" {
			line := fmt.Sprintf("Pool %s is %s", p.Name, state)
			if s != nil && s.Status != "" {
				line += ": " + strings.TrimSuffix(s.Status, ".")
			}
			if s != nil && s.Action != "" {
				line += ". Action: " + strings.TrimSuffix(s.Action, ".")
			}
			found = append(found, line+".")
		}
		switch {
		case p.Capacity > CapacityFail:
			found = append(found, fmt.Sprintf("Pool %s is %d %% full, past %d %%: %s free.", p.Name, p.Capacity, CapacityFail, bytesSaid(p.Free)))
		case p.Capacity > CapacityWarn:
			warn = append(warn, fmt.Sprintf("Pool %s is %d %% full, past %d %%: %s free.", p.Name, p.Capacity, CapacityWarn, bytesSaid(p.Free)))
		}
		if s == nil {
			found = append(found, fmt.Sprintf("zpool status said nothing of pool %s.", p.Name))
			continue
		}
		walk(s.Root, func(d *Device, _ int) {
			if d.Kind == "section" {
				return
			}
			fault, past := mem.errorFinding(p.Name, d, l.Now)
			if fault != "" {
				found = append(found, sentence(fault))
			}
			if past != "" {
				warn = append(warn, sentence(past))
			}
			if d.Kind != "pool" && d.State != "" && d.State != "ONLINE" && d.State != "AVAIL" && d.State != "INUSE" {
				// A device not online already makes its pool DEGRADED or worse; said here by name.
				found = append(found, fmt.Sprintf("%s %s in pool %s is %s.", d.Kind, d.Name, p.Name, d.State))
			}
		}, 0)
		if e := strings.TrimSpace(s.Errors); e != "" && e != "No known data errors" {
			found = append(found, fmt.Sprintf("Pool %s: %s.", p.Name, strings.TrimSuffix(e, ".")))
		}
		for _, r := range s.Resilvers() {
			what := "the resilver"
			if r.Vdev != "" {
				what += " of " + r.Vdev
			}
			switch {
			case r.State == "running":
				line := fmt.Sprintf("A resilver")
				if r.Vdev != "" {
					line += " of " + r.Vdev
				}
				line += fmt.Sprintf(" in pool %s is running", p.Name)
				if r.Done != nil {
					line += fmt.Sprintf(": %.2f %% done", *r.Done)
				}
				if r.ToGo != "" {
					line += ", " + r.ToGo + " to go"
				}
				warn = append(warn, sentence(strings.ToLower(line[:1])+line[1:]))
			case r.State == "finished" && deref(r.Errors) > 0:
				found = append(found, sentence(fmt.Sprintf("%s in pool %s finished with %d errors, %s", what, p.Name, deref(r.Errors), stamp(r.Ended))))
			}
		}
		late, by := ScrubLate(p.Name, l.Schedule)
		ended := LastScrubEnded(*s)
		switch {
		case ended != nil && l.Now.Sub(*ended) > late:
			warn = append(warn, fmt.Sprintf("Pool %s was last scrubbed %.0f days ago, more than %s (%d days).", p.Name,
				l.Now.Sub(*ended).Hours()/24, by, int(late.Hours()/24)))
		case len(s.Scans) == 0:
			warn = append(warn, fmt.Sprintf("Pool %s was never scrubbed.", p.Name))
		}
		if l.Schedule != nil && l.Schedule.Pool == p.Name {
			warn = append(warn, scheduleFindings(l.Schedule, s, l.Now, l.Served)...)
		}
	}
	if l.Schedule != nil && statusOf(l.Status, l.Schedule.Pool) == nil {
		warn = append(warn, fmt.Sprintf("The scrub schedule names pool %s, which is not imported here.", l.Schedule.Pool))
	}
	mem.save()
	if len(found) > 0 {
		return Health{Healthy: false, Why: strings.Join(found, " "), Found: found, Warnings: warn}
	}
	var parts []string
	for _, p := range l.Pools {
		parts = append(parts, fmt.Sprintf("%s ONLINE, %d %% full", p.Name, p.Capacity))
	}
	why := "Every pool is online with no new errors: " + strings.Join(parts, "; ") + "."
	if len(warn) > 0 {
		why += " Warnings: " + strings.Join(warn, " ")
	}
	return Health{Healthy: true, Why: why, Warnings: warn}
}

// Health reads the pools and judges them. A machine without ZFS is unhealthy, loudly: the module was put
// where it does not apply.
func (m *Machine) Health(ctx context.Context) Health {
	pools, err := m.Pools(ctx)
	if err != nil {
		if a, ok := err.(errAbsent); ok {
			return Health{Healthy: false, Why: sentence(a.why) + " Unassign the module from this machine.", Found: []string{a.why}}
		}
		return Health{Healthy: false, Why: sentence("zpool list did not answer: " + err.Error()), Found: []string{err.Error()}}
	}
	status, err := m.Status(ctx)
	if err != nil {
		return Health{Healthy: false, Why: sentence("zpool status did not answer: " + err.Error()), Found: []string{err.Error()}}
	}
	sched, serr := m.ScheduleNow(ctx)
	return m.Errors.Judge(Look{Pools: pools, Status: status, Now: m.Now(), Schedule: sched, ScheduleErr: serr,
		Served: m.readServed(ServedFile)})
}

func sentence(s string) string {
	s = strings.TrimSpace(s)
	if s == "" {
		return s
	}
	s = strings.ToUpper(s[:1]) + s[1:]
	if !strings.HasSuffix(s, ".") {
		s += "."
	}
	return s
}