mesh-catalog/modules/zfs/cmd/zfs-tools/health.go

package main

import (
	"context"
	"fmt"
	"strings"
	"time"
)

// The lines a pool's capacity is judged against: past CapacityWarn a warning, past CapacityFail
// unhealthy. ZFS slows as a pool fills, and a full pool can refuse even a deletion.
const (
	CapacityWarn = 80
	CapacityFail = 90
)

// Health is the module's health, as the node-engine asks it (novox/hq ADR 0240): unhealthy with why, one
// sentence per finding; warnings do not make it unhealthy.
type Health struct {
	Healthy  bool     `json:"healthy"`
	Why      string   `json:"why"`
	Found    []string `json:"found,omitempty"`
	Warnings []string `json:"warnings,omitempty"`
}

// Judge judges the pools from what zpool list and zpool status said.
func Judge(pools []Pool, status []PoolStatus, now time.Time) Health {
	var found, warn []string
	if len(pools) == 0 {
		found = append(found, "ZFS is on this machine and no pool is imported.")
	}
	for _, p := range pools {
		s := statusOf(status, p.Name)
		state := p.Health
		if s != nil && s.State != "" {
			state = s.State
		}
		if state != "ONLINE" {
			line := fmt.Sprintf("Pool %s is %s", p.Name, state)
			if s != nil && s.Status != "" {
				line += ": " + strings.TrimSuffix(s.Status, ".")
			}
			if s != nil && s.Action != "" {
				line += ". Action: " + strings.TrimSuffix(s.Action, ".")
			}
			found = append(found, line+".")
		}
		switch {
		case p.Capacity > CapacityFail:
			found = append(found, fmt.Sprintf("Pool %s is %d %% full, past %d %%: %s free.", p.Name, p.Capacity, CapacityFail, bytesSaid(p.Free)))
		case p.Capacity > CapacityWarn:
			warn = append(warn, fmt.Sprintf("Pool %s is %d %% full, past %d %%: %s free.", p.Name, p.Capacity, CapacityWarn, bytesSaid(p.Free)))
		}
		if s == nil {
			found = append(found, fmt.Sprintf("zpool status said nothing of pool %s.", p.Name))
			continue
		}
		walk(s.Root, func(d *Device, _ int) {
			if d.Kind == "section" {
				return
			}
			if d.Errors() > 0 {
				found = append(found, fmt.Sprintf("%s %s in pool %s has errors: read %d, write %d, checksum %d.", d.Kind, d.Name, p.Name, d.Read, d.Write, d.Checksum))
			}
			if d.Kind != "pool" && d.State != "" && d.State != "ONLINE" && d.State != "AVAIL" && d.State != "INUSE" {
				// A device not online already makes its pool DEGRADED or worse; said here by name.
				found = append(found, fmt.Sprintf("%s %s in pool %s is %s.", d.Kind, d.Name, p.Name, d.State))
			}
		}, 0)
		if e := strings.TrimSpace(s.Errors); e != "" && e != "No known data errors" {
			found = append(found, fmt.Sprintf("Pool %s: %s.", p.Name, strings.TrimSuffix(e, ".")))
		}
		if ended := LastScrubEnded(s.Scan); ended != nil && now.Sub(*ended) > ScrubWarnAfter {
			warn = append(warn, fmt.Sprintf("Pool %s was last scrubbed %.0f days ago, more than %d.", p.Name, now.Sub(*ended).Hours()/24, int(ScrubWarnAfter.Hours()/24)))
		} else if s.Scan.State == "none" {
			warn = append(warn, fmt.Sprintf("Pool %s was never scrubbed.", p.Name))
		}
	}
	if len(found) > 0 {
		return Health{Healthy: false, Why: strings.Join(found, " "), Found: found, Warnings: warn}
	}
	var parts []string
	for _, p := range pools {
		parts = append(parts, fmt.Sprintf("%s ONLINE, %d %% full", p.Name, p.Capacity))
	}
	why := "Every pool is online with no errors: " + strings.Join(parts, "; ") + "."
	if len(warn) > 0 {
		why += " Warnings: " + strings.Join(warn, " ")
	}
	return Health{Healthy: true, Why: why, Warnings: warn}
}

// Health reads the pools and judges them. A machine without ZFS is unhealthy, loudly: the module was put
// where it does not apply.
func (m *Machine) Health(ctx context.Context) Health {
	pools, err := m.Pools(ctx)
	if err != nil {
		if a, ok := err.(errAbsent); ok {
			return Health{Healthy: false, Why: sentence(a.why) + " Unassign the module from this machine.", Found: []string{a.why}}
		}
		return Health{Healthy: false, Why: sentence("zpool list did not answer: " + err.Error()), Found: []string{err.Error()}}
	}
	status, err := m.Status(ctx)
	if err != nil {
		return Health{Healthy: false, Why: sentence("zpool status did not answer: " + err.Error()), Found: []string{err.Error()}}
	}
	return Judge(pools, status, m.Now())
}

func sentence(s string) string {
	s = strings.TrimSpace(s)
	if s == "" {
		return s
	}
	s = strings.ToUpper(s[:1]) + s[1:]
	if !strings.HasSuffix(s, ".") {
		s += "."
	}
	return s
}