mesh-catalog/modules/sensors/cmd/sensors/drives.go

package main

import (
	"context"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"
)

// Drives and their health. Which drives the machine has is read from /sys by anyone; what each says of its
// own health (the NVMe health log, SATA and SAS SMART) is read by the module's drive reader, as root, with
// smartctl, and left in DrivesPath for the tools (novox/hq ADR 0256).

// The two files the module's processes write: the drive reader's (DrivesPath) and the fan watch's
// (FansPath). Each process writes only its own.
const (
	DrivesPath = "/run/mesh-sensors/drives.json"
	FansPath   = "/run/mesh-sensors/fans.json"
)

// AsleepHighAfter is how long a drive may be found asleep at every look before the tools say it as HIGH:
// its health has not been read for that long, and a drive that stops answering can look asleep. It never
// makes the module unhealthy on its own (ADR 0256): a drive kept for backups may sleep for weeks.
const AsleepHighAfter = 24 * time.Hour

// StaleAfter is how old either process's file may be before the module is unhealthy: each writes its own
// every FanEvery, whatever a drive does, so ten writes missed is a process that is hung or gone.
const StaleAfter = 5 * time.Minute

// Smartctl is the program, by its path: the watch runs as root and runs nothing found on a PATH.
const Smartctl = "/usr/bin/smartctl"

// FlagFailing is said beside a drive that says it is failing.
const FlagFailing = "FAILING"

// Readings is one process's file: the drive reader fills Drives, the fan watch FansRan.
type Readings struct {
	BootID    string                  `json:"boot_id"`
	WrittenAt time.Time               `json:"written_at"`
	Drives    map[string]DriveReading `json:"drives"`
	// FansRan is when the watch last saw each fan turning, by its key (asus/fan1), since the machine started.
	FansRan map[string]time.Time `json:"fans_ran"`
}

// DriveReading is one drive as smartctl last answered for it.
type DriveReading struct {
	ReadAt   time.Time       `json:"read_at"`
	Smartctl json.RawMessage `json:"smartctl,omitempty"`
	Status   int             `json:"status"`
	// Asleep says the drive was in standby at the newest look, so it was not woken and the reading is the
	// one before.
	Asleep  bool   `json:"asleep,omitempty"`
	NotRead string `json:"not_read,omitempty"`
	// LookedAt is the newest look at the drive, asleep or not: what the next look is timed from.
	LookedAt time.Time `json:"looked_at"`
	// AsleepSince is the first of the looks in a row that found the drive asleep.
	AsleepSince *time.Time `json:"asleep_since,omitempty"`
	// InFlightSince is when the read under way began: a read that never ends (a drive or a controller that
	// stopped answering, a smartctl stuck in the kernel) is seen by how old this is.
	InFlightSince *time.Time `json:"in_flight_since,omitempty"`
}

// Drive is one drive, as the tools say it.
type Drive struct {
	Device     string  `json:"device"`
	Model      string  `json:"model,omitempty"`
	Serial     string  `json:"serial,omitempty"`
	Firmware   string  `json:"firmware,omitempty"`
	SizeGiB    float64 `json:"size_gib"`
	Rotational bool    `json:"rotational"`
	Transport  string  `json:"transport"`
	// Health is "passed", "failing" or "not read".
	Health   string   `json:"health"`
	Flag     string   `json:"flag,omitempty"`
	Findings []string `json:"findings,omitempty"`

	TemperatureC     *int64   `json:"temperature_c,omitempty"`
	PowerOnHours     *int64   `json:"power_on_hours,omitempty"`
	PercentageUsed   *int64   `json:"percentage_used,omitempty"`
	AvailableSpare   *int64   `json:"available_spare_pct,omitempty"`
	SpareThreshold   *int64   `json:"available_spare_threshold_pct,omitempty"`
	MediaErrors      *int64   `json:"media_errors,omitempty"`
	CriticalWarning  *int64   `json:"critical_warning,omitempty"`
	Reallocated      *int64   `json:"reallocated_sectors,omitempty"`
	Pending          *int64   `json:"pending_sectors,omitempty"`
	Uncorrectable    *int64   `json:"offline_uncorrectable,omitempty"`
	ErrorLogCount    *int64   `json:"error_log_count,omitempty"`
	LastSelfTest     string   `json:"last_self_test,omitempty"`
	ReadAt           string   `json:"read_at,omitempty"`
	Asleep           bool     `json:"asleep_at_last_look,omitempty"`
	AsleepSince      string   `json:"asleep_since,omitempty"`
	NotRead          string   `json:"not_read,omitempty"`
	failingSentences []string // why it is failing, each a clause for the operator
}

// Name is how a person reads which drive this is.
func (d Drive) Name() string {
	if d.Model != "" {
		return fmt.Sprintf("the drive %s (%s)", d.Device, d.Model)
	}
	return "the drive " + d.Device
}

// notDrives are block devices that are not a drive of their own: loop files, memory, device-mapper and
// software RAID volumes (built on drives that are listed themselves), optical and floppy drives, network
// block devices.
var notDrives = []string{"loop", "ram", "zram", "dm-", "md", "sr", "fd", "nbd", "rbd"}

// BlockDrives lists the machine's drives from /sys/block: whole devices backed by hardware.
func (m *Machine) BlockDrives() []Drive {
	var out []Drive
	for _, b := range m.names("/sys/block") {
		skip := strings.HasSuffix(b, "boot0") || strings.HasSuffix(b, "boot1") || strings.HasSuffix(b, "rpmb")
		for _, p := range notDrives {
			skip = skip || strings.HasPrefix(b, p)
		}
		if skip || !m.exists("/sys/block/"+b+"/device") {
			continue
		}
		dir := "/sys/block/" + b
		d := Drive{Device: b, Model: m.read(dir + "/device/model"), Transport: transport(b, m.read(dir+"/device/vendor"))}
		if where, err := filepath.EvalSymlinks(m.path(dir)); err == nil && strings.Contains(where, "/usb") {
			d.Transport = "usb"
		}
		if s, ok := m.readInt(dir + "/size"); ok {
			d.SizeGiB = round1(float64(s) * 512 / (1 << 30))
		}
		if r, ok := m.readInt(dir + "/queue/rotational"); ok {
			d.Rotational = r == 1
		}
		out = append(out, d)
	}
	return out
}

func transport(name, vendor string) string {
	switch {
	case strings.HasPrefix(name, "nvme"):
		return "nvme"
	case strings.HasPrefix(name, "vd"), strings.HasPrefix(name, "xvd"):
		return "virtual"
	case strings.HasPrefix(name, "mmcblk"):
		return "mmc"
	case strings.TrimSpace(vendor) == "ATA":
		return "ata"
	}
	return "scsi"
}

// who names the process that writes a file, as a person reads it.
func who(path string) string {
	if path == DrivesPath {
		return "the module's drive reader"
	}
	return "the module's fan watch"
}

// ReadReadings is one process's file, or why there is none to use.
func (m *Machine) ReadReadings(path string) (*Readings, string) {
	raw, err := os.ReadFile(m.path(path))
	if err != nil {
		return nil, who(path) + " has left no readings (is it running?)"
	}
	var r Readings
	if err := json.Unmarshal(raw, &r); err != nil {
		return nil, who(path) + " left readings this tool cannot read: " + err.Error()
	}
	if id := m.bootID(); id != "" && r.BootID != "" && r.BootID != id {
		return nil, who(path) + " has not written anything since the machine started"
	}
	return &r, ""
}

// Drives is every drive with its health as the watch's readings say it, judged. r is nil, with why, when
// there are none to use.
func (m *Machine) Drives(r *Readings, why string) ([]Drive, string) {
	drives := m.BlockDrives()
	note := why
	if r != nil && m.Now().Sub(r.WrittenAt) > StaleAfter {
		note = fmt.Sprintf("the module's drive reader last wrote %s ago; it may not be running", ago(m.Now().Sub(r.WrittenAt)))
	}
	for i := range drives {
		d := &drives[i]
		if r == nil {
			d.Health, d.NotRead = "not read", why
			continue
		}
		reading, ok := r.Drives[d.Device]
		if !ok {
			d.Health, d.NotRead = "not read", "the module's watch has not read this drive yet"
			continue
		}
		judgeDrive(d, reading, m.Now())
	}
	return drives, note
}

func ago(d time.Duration) string {
	switch {
	case d < time.Minute:
		return fmt.Sprintf("%d s", int(d.Seconds()))
	case d < 2*time.Hour:
		return fmt.Sprintf("%d min", int(d.Minutes()))
	}
	return fmt.Sprintf("%.1f h", d.Hours())
}

// smartctl's JSON, the parts the module reads (smartmontools 7).
type smartJSON struct {
	Smartctl struct {
		Messages []struct {
			String   string `json:"string"`
			Severity string `json:"severity"`
		} `json:"messages"`
	} `json:"smartctl"`
	ModelName       string `json:"model_name"`
	SerialNumber    string `json:"serial_number"`
	FirmwareVersion string `json:"firmware_version"`
	SmartSupport    *struct {
		Available bool `json:"available"`
		Enabled   bool `json:"enabled"`
	} `json:"smart_support"`
	SmartStatus *struct {
		Passed bool `json:"passed"`
	} `json:"smart_status"`
	PowerMode *struct {
		Name string `json:"name"`
	} `json:"power_mode"`
	NVMe *struct {
		CriticalWarning         int64 `json:"critical_warning"`
		AvailableSpare          int64 `json:"available_spare"`
		AvailableSpareThreshold int64 `json:"available_spare_threshold"`
		PercentageUsed          int64 `json:"percentage_used"`
		MediaErrors             int64 `json:"media_errors"`
		PowerOnHours            int64 `json:"power_on_hours"`
	} `json:"nvme_smart_health_information_log"`
	ATA *struct {
		Table []struct {
			ID         int    `json:"id"`
			Name       string `json:"name"`
			Value      int64  `json:"value"`
			Thresh     int64  `json:"thresh"`
			WhenFailed string `json:"when_failed"`
			Flags      struct {
				Prefailure bool `json:"prefailure"`
			} `json:"flags"`
			Raw struct {
				Value int64 `json:"value"`
			} `json:"raw"`
		} `json:"table"`
	} `json:"ata_smart_attributes"`
	ErrorLog *struct {
		Summary struct {
			Count int64 `json:"count"`
		} `json:"summary"`
	} `json:"ata_smart_error_log"`
	SelfTests *struct {
		Standard struct {
			Table []struct {
				Type struct {
					String string `json:"string"`
				} `json:"type"`
				Status struct {
					String string `json:"string"`
					Passed *bool  `json:"passed"`
				} `json:"status"`
			} `json:"table"`
		} `json:"standard"`
	} `json:"ata_smart_self_test_log"`
	Temperature *struct {
		Current int64 `json:"current"`
	} `json:"temperature"`
	PowerOnTime *struct {
		Hours int64 `json:"hours"`
	} `json:"power_on_time"`
}

// nvmeWarnings are the NVMe health log's critical warning bits, as a person reads each.
var nvmeWarnings = []struct {
	bit  int64
	said string
}{
	{0x01, "its spare capacity is below its threshold"},
	{0x02, "its temperature is past one of its own thresholds"},
	{0x04, "its reliability is degraded by media or internal errors"},
	{0x08, "it has put itself in read-only mode"},
	{0x10, "its volatile memory backup has failed"},
	{0x20, "its persistent memory region has become read-only"},
}

func num(v int64) *int64 { return &v }

// judgeDrive reads one drive's smartctl answer into the drive, with its findings: FAILING when the drive
// says so itself, HIGH for the early signs a drive gives before it fails.
func judgeDrive(d *Drive, reading DriveReading, now time.Time) {
	if !reading.ReadAt.IsZero() {
		d.ReadAt = reading.ReadAt.UTC().Format(time.RFC3339)
	}
	d.Asleep = reading.Asleep
	var asleepLong string
	if reading.Asleep && reading.AsleepSince != nil {
		d.AsleepSince = reading.AsleepSince.UTC().Format(time.RFC3339)
		if for_ := now.Sub(*reading.AsleepSince); for_ >= AsleepHighAfter {
			asleepLong = fmt.Sprintf("it has been asleep at every look for %s, so its health has not been read since; "+
				"a drive that stopped answering can look asleep", ago(for_))
		}
	}
	defer func() {
		if asleepLong != "" {
			d.Findings = append(d.Findings, asleepLong)
			if d.Flag == "" {
				d.Flag = FlagHigh
			}
		}
	}()
	var s smartJSON
	if len(reading.Smartctl) == 0 || json.Unmarshal(reading.Smartctl, &s) != nil {
		d.Health, d.NotRead = "not read", reading.NotRead
		if d.NotRead == "" {
			d.NotRead = "smartctl gave no answer for it"
		}
		return
	}
	if s.ModelName != "" {
		d.Model = s.ModelName
	}
	d.Serial, d.Firmware = s.SerialNumber, s.FirmwareVersion
	if s.SmartStatus == nil && s.NVMe == nil {
		d.Health = "not read"
		switch {
		case s.SmartSupport != nil && !s.SmartSupport.Available:
			d.NotRead = "the drive does not support SMART"
		case s.SmartSupport != nil && !s.SmartSupport.Enabled:
			d.NotRead = "SMART is turned off on the drive"
		default:
			d.NotRead = firstError(s, reading.NotRead)
		}
		return
	}
	d.Health = "passed"
	var failing, high []string
	// An NVMe drive's overall verdict is its critical warning byte, so it is judged from the bits below;
	// the temperature bit alone is a drive running hot, not one failing.
	if s.SmartStatus != nil && !s.SmartStatus.Passed && s.NVMe == nil {
		failing = append(failing, "its overall SMART health check failed")
	}
	if s.Temperature != nil && s.Temperature.Current != 0 {
		d.TemperatureC = num(s.Temperature.Current)
	}
	if s.PowerOnTime != nil {
		d.PowerOnHours = num(s.PowerOnTime.Hours)
	}
	if n := s.NVMe; n != nil {
		d.CriticalWarning, d.PercentageUsed = num(n.CriticalWarning), num(n.PercentageUsed)
		d.AvailableSpare, d.SpareThreshold, d.MediaErrors = num(n.AvailableSpare), num(n.AvailableSpareThreshold), num(n.MediaErrors)
		if d.PowerOnHours == nil {
			d.PowerOnHours = num(n.PowerOnHours)
		}
		for _, w := range nvmeWarnings {
			switch {
			case n.CriticalWarning&w.bit == 0:
			case w.bit == 0x02:
				high = append(high, w.said)
			default:
				failing = append(failing, w.said)
			}
		}
		if n.MediaErrors > 0 {
			high = append(high, fmt.Sprintf("it has had %d media errors", n.MediaErrors))
		}
		if n.PercentageUsed >= 90 {
			high = append(high, fmt.Sprintf("it has used %d %% of its rated endurance", n.PercentageUsed))
		}
	}
	if a := s.ATA; a != nil {
		for _, at := range a.Table {
			switch at.ID {
			case 5:
				d.Reallocated = num(at.Raw.Value)
			case 197:
				d.Pending = num(at.Raw.Value)
			case 198:
				d.Uncorrectable = num(at.Raw.Value)
			}
			switch at.WhenFailed {
			case "now":
				// A pre-failure attribute at its threshold is the drive saying it is about to fail; an
				// old-age one (a temperature, a count of hours) is wear or a hot day.
				said := fmt.Sprintf("its attribute %s is at %d, at or below its own threshold of %d", at.Name, at.Value, at.Thresh)
				if at.Flags.Prefailure {
					failing = append(failing, said)
				} else {
					high = append(high, said+" (an old-age attribute)")
				}
			case "past":
				high = append(high, fmt.Sprintf("its attribute %s was at or below its own threshold in the past", at.Name))
			}
		}
		for _, c := range []struct {
			v    *int64
			said string
		}{{d.Reallocated, "sectors reallocated"}, {d.Pending, "sectors waiting to be reallocated"}, {d.Uncorrectable, "sectors it could not read"}} {
			if c.v != nil && *c.v > 0 {
				high = append(high, fmt.Sprintf("it has %d %s", *c.v, c.said))
			}
		}
	}
	if s.ErrorLog != nil {
		d.ErrorLogCount = num(s.ErrorLog.Summary.Count)
		if s.ErrorLog.Summary.Count > 0 {
			high = append(high, fmt.Sprintf("its error log holds %d errors", s.ErrorLog.Summary.Count))
		}
	}
	if s.SelfTests != nil && len(s.SelfTests.Standard.Table) > 0 {
		last := s.SelfTests.Standard.Table[0]
		d.LastSelfTest = strings.TrimSpace(last.Type.String + ": " + last.Status.String)
		if last.Status.Passed != nil && !*last.Status.Passed {
			high = append(high, "its last self-test did not pass ("+last.Status.String+")")
		}
	}
	d.failingSentences = failing
	switch {
	case len(failing) > 0:
		d.Health, d.Flag = "failing", FlagFailing
		d.Findings = append(d.Findings, failing...)
	case len(high) > 0:
		d.Flag = FlagHigh
	}
	d.Findings = append(d.Findings, high...)
}

func firstError(s smartJSON, fallback string) string {
	for _, msg := range s.Smartctl.Messages {
		if msg.Severity == "error" && strings.TrimSpace(msg.String) != "" {
			return "smartctl: " + strings.TrimSpace(msg.String)
		}
	}
	if fallback != "" {
		return fallback
	}
	return "smartctl gave no health for it"
}

// deviceType is smartctl's -d for a drive whose kind is known, so smartctl does not probe for it: probing
// a drive behind a USB bridge can spin it up. Empty for a drive smartctl must find out about itself.
func deviceType(d Drive) string {
	switch d.Transport {
	case "nvme":
		return "nvme"
	case "ata", "usb":
		return "sat"
	case "scsi":
		return "scsi"
	}
	return ""
}

// ReadDrive asks smartctl for one drive's health, as root (directly when this process is root, else
// through sudo -n), without waking a drive in standby. `-n standby,2,3` makes smartctl stop with status 2
// when the drive is in standby or asleep (an ATA drive with its power mode, a SAS drive or one the
// operating system powered down with its own words), and with status 3 when the drive cannot say (a USB
// bridge that does not pass the question on): such a drive is not read, because reading it could wake it.
// A drive asleep keeps the reading before, and since when it has been asleep.
func (m *Machine) ReadDrive(d Drive, before *DriveReading) DriveReading {
	args := []string{"--json=c", "-a"}
	if t := deviceType(d); t != "" {
		args = append(args, "-d", t)
	}
	args = append(args, "-n", "standby,2,3", "/dev/"+d.Device)
	program := Smartctl
	if m.UID != 0 {
		program, args = "sudo", append([]string{"-n", Smartctl}, args...)
	}
	r := m.Run(context.Background(), program, args...)
	now := m.Now()
	out := DriveReading{ReadAt: now, LookedAt: now, Status: r.Status}
	switch {
	case r.Err == "ENOENT" && program == Smartctl:
		out.NotRead = "smartctl is not installed; the module declares smartmontools, so the package has not been applied yet"
		return out
	case r.Err == "ENOENT":
		out.NotRead = "the drive reader is not root and sudo is not installed to read the drive as root"
		return out
	case r.Err != "":
		out.NotRead = "smartctl did not answer: " + r.Err
		return out
	case program == "sudo" && strings.HasPrefix(strings.TrimSpace(r.Stderr), "sudo:") && strings.Contains(r.Stderr, "command not found"):
		out.NotRead = "smartctl is not installed; the module declares smartmontools, so the package has not been applied yet"
		return out
	case program == "sudo" && strings.HasPrefix(strings.TrimSpace(r.Stderr), "sudo:"):
		out.NotRead = "the drive reader is not root and may not run smartctl through sudo without a prompt"
		return out
	}
	var s smartJSON
	parsed := json.Unmarshal([]byte(r.Stdout), &s) == nil
	switch {
	case parsed && r.Status == 2 && asleep(s):
		since := now
		kept := DriveReading{NotRead: "the drive was asleep at every look, and is not woken to be read"}
		if before != nil {
			kept = *before
			if before.Asleep && before.AsleepSince != nil {
				since = *before.AsleepSince
			}
		}
		kept.Asleep, kept.LookedAt, kept.AsleepSince, kept.InFlightSince = true, now, &since, nil
		return kept
	case parsed && r.Status == 3 && saysNoPowerMode(s):
		out.NotRead = "the drive cannot say whether it is asleep (a USB bridge, usually), so it is not read, never to wake it"
		return out
	case !parsed:
		out.NotRead = "smartctl answered something other than JSON (status " + strconv.Itoa(r.Status) + ")"
		return out
	}
	out.Smartctl = json.RawMessage(strings.TrimSpace(r.Stdout))
	return out
}

// asleep says smartctl stopped because the drive is in standby or asleep: by the ATA power mode it
// parsed, or by its own words "Device is in <mode> mode, exit(2)", which it says for a SAS drive and for
// a drive the operating system powered down (ataprint.cpp, scsiprint.cpp, smartctl.cpp of smartmontools
// 7.5). A failure to open the drive says neither, whatever else its words hold.
func asleep(s smartJSON) bool {
	if s.PowerMode != nil {
		switch s.PowerMode.Name {
		case "STANDBY", "STANDBY_Y", "SLEEP":
			return true
		}
	}
	for _, msg := range s.Smartctl.Messages {
		if sleepWords.MatchString(strings.TrimSpace(msg.String)) {
			return true
		}
	}
	return false
}

// sleepWords are smartctl's words when -n standby stopped it: an ATA mode (STANDBY, SLEEP), a SAS mode,
// which holds spaces (STANDBY BY TIMER, STANDBY_Y BY COMMAND, LOW POWER), or STANDBY (OS). An idle mode
// never stops smartctl under -n standby, so it never appears here.
var sleepWords = regexp.MustCompile(`^Device is in [A-Z_]+( [A-Z_]+)*( \(OS\))? mode, exit\(2\)$`)

func saysNoPowerMode(s smartJSON) bool {
	for _, msg := range s.Smartctl.Messages {
		if strings.HasPrefix(msg.String, "CHECK POWER MODE not implemented") || strings.HasPrefix(msg.String, "CHECK POWER MODE returned unknown value") {
			return true
		}
	}
	return false
}

// WriteReadings writes one process's file whole, through a rename, readable by every account: it holds
// no secret, and the tools read it as the operator account.
func (m *Machine) WriteReadings(file string, r Readings) error {
	path := m.path(file)
	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
		return err
	}
	// MkdirAll leaves a directory that was already there as it was: the tools must be able to enter it.
	if err := os.Chmod(filepath.Dir(path), 0o755); err != nil {
		return err
	}
	raw, err := json.Marshal(r)
	if err != nil {
		return err
	}
	tmp := path + ".new"
	if err := os.WriteFile(tmp, raw, 0o644); err != nil {
		return err
	}
	if err := os.Chmod(tmp, 0o644); err != nil {
		return err
	}
	return os.Rename(tmp, path)
}