mesh-catalog/modules/sensors/cmd/sensors/findings.go

package main

import (
	"fmt"
	"sort"
	"strings"
)

// What the module finds, said two ways: a line per finding for whoever asks (sensors_summary), and the
// module's health for the node-engine (sensors_health, novox/hq ADR 0240 and ADR 0256 rule 4).

// Look is one reading of every class.
type Look struct {
	Temperatures []Temperature
	Fans         []Fan
	Drives       []Drive
	DrivesNote   string
	Faults       Faults
	// Stale is why the module's own processes are not to be trusted now: a file missing, from an earlier
	// boot or older than StaleAfter, a drive's read stuck, a drive not looked at for twice its cadence.
	Stale []string
}

// LookNow reads every class and each process's file once, and judges the fans against what the fan watch
// saw turning since the machine started, and the drive reader by how old its file and its reads are.
func (m *Machine) LookNow() Look {
	l := Look{Temperatures: m.Temperatures(), Fans: m.Fans(), Faults: m.Faults()}
	now := m.Now()
	dr, dwhy := m.ReadReadings(DrivesPath)
	l.Drives, l.DrivesNote = m.Drives(dr, dwhy)
	fr, fwhy := m.ReadReadings(FansPath)
	for _, f := range []struct {
		r    *Readings
		why  string
		path string
	}{{dr, dwhy, DrivesPath}, {fr, fwhy, FansPath}} {
		switch {
		case f.r == nil:
			l.Stale = append(l.Stale, capital(f.why)+".")
		case now.Sub(f.r.WrittenAt) > StaleAfter:
			l.Stale = append(l.Stale, fmt.Sprintf("%s has not written its readings for %s, so it is hung or gone.",
				capital(who(f.path)), ago(now.Sub(f.r.WrittenAt))))
		}
	}
	if dr != nil {
		byName := map[string]Drive{}
		for _, d := range l.Drives {
			byName[d.Device] = d
		}
		for _, name := range sortedKeys(dr.Drives) {
			rd := dr.Drives[name]
			d, present := byName[name]
			switch {
			case rd.InFlightSince != nil && now.Sub(*rd.InFlightSince) > ReadBound():
				l.Stale = append(l.Stale, fmt.Sprintf("The read of the drive %s has not finished for %s; the drive or its "+
					"controller may have stopped answering.", name, ago(now.Sub(*rd.InFlightSince))))
			case present && rd.InFlightSince == nil && !rd.LookedAt.IsZero() && now.Sub(rd.LookedAt) > 2*cadence(d):
				l.Stale = append(l.Stale, fmt.Sprintf("The drive %s has not been looked at for %s, more than twice as long "+
					"as it should be.", name, ago(now.Sub(rd.LookedAt))))
			}
		}
	}
	var ran map[string]bool
	if fr != nil {
		ran = map[string]bool{}
		for k := range fr.FansRan {
			ran[k] = true
		}
	}
	l.Fans = judgeFans(l.Fans, ran, l.Temperatures)
	return l
}

func sortedKeys(m map[string]DriveReading) []string {
	out := make([]string, 0, len(m))
	for k := range m {
		out = append(out, k)
	}
	sort.Strings(out)
	return out
}

// Problems is what makes the module unhealthy: each a sentence for the operator.
func (l Look) Problems() []string {
	var out []string
	for _, t := range l.Temperatures {
		if t.Flag == FlagCritical && t.DrivesHealth() {
			out = append(out, fmt.Sprintf("%s is at %.1f °C, at or above %s of %.1f °C.", capital(t.Name()), *t.C, t.CriticalIs, *t.Critical))
		}
	}
	for _, d := range l.Drives {
		if d.Flag == FlagFailing {
			out = append(out, fmt.Sprintf("%s says it is failing: %s.", capital(d.Name()), joinAnd(d.failingSentences)))
		}
	}
	for _, f := range l.Fans {
		if f.Flag == FlagStopped {
			out = append(out, fmt.Sprintf("%s has stopped: %s.", capital(f.Name()), f.Why))
		}
	}
	for _, c := range l.Faults.Memory {
		if c.Uncorrectable > 0 {
			out = append(out, fmt.Sprintf("The memory reports %d uncorrectable errors since the machine started (memory controller %s).",
				c.Uncorrectable, c.Controller))
		}
	}
	return append(out, l.Stale...)
}

// Health is the answer the node-engine reads from a tool check: healthy or not, with why.
type Health struct {
	Healthy bool     `json:"healthy"`
	Why     string   `json:"why"`
	Found   []string `json:"found,omitempty"`
}

// Health judges the look. Unhealthy only on what the hardware says is failing; HIGH readings and what
// could not be read are said by the other tools and raise nothing.
func (l Look) Health() Health {
	p := l.Problems()
	if len(p) == 0 {
		why := "No temperature is at its critical value, no drive says it is failing, no fan has stopped and no memory error is uncorrectable."
		unread := 0
		for _, d := range l.Drives {
			if d.Health == "not read" {
				unread++
			}
		}
		// Said, never passed as read: a drive asleep since boot, or one the watch cannot read.
		if unread > 0 {
			why += fmt.Sprintf(" The health of %d of %d drives has not been read.", unread, len(l.Drives))
		}
		return Health{Healthy: true, Why: why}
	}
	return Health{Healthy: false, Why: strings.Join(p, " "), Found: p}
}

// Said is one line per finding, flagged, then one line per class saying what was read.
func (l Look) Said() []string {
	var out []string
	for _, t := range l.Temperatures {
		switch t.Flag {
		case FlagCritical:
			out = append(out, fmt.Sprintf("CRITICAL: %s at %.1f °C, at or above %s (%.1f °C)", t.Name(), *t.C, t.CriticalIs, *t.Critical))
		case FlagHigh:
			out = append(out, fmt.Sprintf("HIGH: %s at %.1f °C, at or above %s (%.1f °C)", t.Name(), *t.C, t.HighIs, *t.High))
		}
		if t.Alarm && t.Flag == "" {
			out = append(out, fmt.Sprintf("HIGH: %s raises its chip's alarm at %.1f °C", t.Name(), *t.C))
		}
	}
	for _, d := range l.Drives {
		switch d.Flag {
		case FlagFailing:
			out = append(out, fmt.Sprintf("FAILING: %s: %s", d.Name(), strings.Join(d.Findings, "; ")))
		case FlagHigh:
			out = append(out, fmt.Sprintf("HIGH: %s: %s", d.Name(), strings.Join(d.Findings, "; ")))
		}
	}
	for _, f := range l.Fans {
		if f.Flag == FlagStopped {
			out = append(out, fmt.Sprintf("STOPPED: %s: %s", f.Name(), f.Why))
		}
	}
	for _, c := range l.Faults.Memory {
		switch c.Flag {
		case FlagCritical:
			out = append(out, fmt.Sprintf("CRITICAL: memory controller %s counted %d uncorrectable and %d corrected errors since the machine started", c.Controller, c.Uncorrectable, c.Corrected))
		case FlagHigh:
			out = append(out, fmt.Sprintf("HIGH: memory controller %s corrected %d errors since the machine started", c.Controller, c.Corrected))
		}
	}
	for _, e := range l.Faults.PCI {
		if e.Flag != "" {
			out = append(out, fmt.Sprintf("HIGH: the PCIe device %s (%s) counted %d fatal and %d non-fatal errors since the machine started", e.Device, orUnknown(e.Driver), e.Fatal, e.NonFatal))
		}
	}
	for _, st := range l.Stale {
		out = append(out, "STALE: "+st)
	}
	return append(out, l.overview()...)
}

// overview is a line per class: how many were read, the hottest, and what is not present.
func (l Look) overview() []string {
	var out []string
	read, notRead := 0, 0
	var hot *Temperature
	for i, t := range l.Temperatures {
		if t.C == nil {
			notRead++
			continue
		}
		read++
		if hot == nil || *t.C > *hot.C {
			hot = &l.Temperatures[i]
		}
	}
	switch {
	case read == 0 && notRead == 0:
		out = append(out, "temperatures: not present (no hwmon chip and no thermal zone reports one)")
	case read == 0:
		out = append(out, fmt.Sprintf("temperatures: %d sensors, none gave a reading", notRead))
	default:
		line := fmt.Sprintf("temperatures: %d read; the hottest is %s at %.1f °C", read, hot.Name(), *hot.C)
		switch {
		case hot.Critical != nil:
			line += fmt.Sprintf(" (critical at %.1f °C)", *hot.Critical)
		case hot.High != nil:
			line += fmt.Sprintf(" (high at %.1f °C)", *hot.High)
		default:
			line += ", which reports no threshold of its own"
		}
		if notRead > 0 {
			line += fmt.Sprintf("; %d gave no reading", notRead)
		}
		out = append(out, line)
	}
	if len(l.Fans) == 0 {
		out = append(out, "fans: not present (no hwmon chip reports a fan)")
	} else {
		var parts []string
		for _, f := range l.Fans {
			if f.RPM == nil {
				parts = append(parts, f.Label+" not read")
				continue
			}
			parts = append(parts, fmt.Sprintf("%s %d RPM", f.Label, *f.RPM))
		}
		out = append(out, "fans: "+strings.Join(parts, ", "))
	}
	if len(l.Drives) == 0 {
		out = append(out, "drives: not present (no drive backed by hardware)")
	} else {
		passed, failing, unread := 0, 0, 0
		for _, d := range l.Drives {
			switch d.Health {
			case "passed":
				passed++
			case "failing":
				failing++
			default:
				unread++
			}
		}
		line := fmt.Sprintf("drives: %d, %d passed, %d failing", len(l.Drives), passed, failing)
		if unread > 0 {
			line += fmt.Sprintf(", %d not read", unread)
			for _, d := range l.Drives {
				if d.NotRead != "" {
					line += " (" + d.Device + ": " + d.NotRead + ")"
					break
				}
			}
		}
		if l.DrivesNote != "" && unread < len(l.Drives) {
			line += "; " + l.DrivesNote
		}
		out = append(out, line)
	}
	errs := "hardware errors: "
	if len(l.Faults.Memory) == 0 {
		errs += "memory not present"
	} else {
		var ce, ue int64
		for _, c := range l.Faults.Memory {
			ce, ue = ce+c.Corrected, ue+c.Uncorrectable
		}
		errs += fmt.Sprintf("memory %d corrected, %d uncorrectable", ce, ue)
	}
	if l.Faults.PCIDevicesCounting == 0 {
		errs += "; PCIe not present"
	} else {
		errs += fmt.Sprintf("; PCIe %d of %d devices counted an error", len(l.Faults.PCI), l.Faults.PCIDevicesCounting)
	}
	if l.Faults.ThrottledCore != nil {
		errs += fmt.Sprintf("; the CPU was slowed for heat %d times", *l.Faults.ThrottledCore)
	}
	return append(out, errs+", since the machine started")
}

func capital(s string) string {
	if s == "" {
		return s
	}
	return strings.ToUpper(s[:1]) + s[1:]
}

func joinAnd(parts []string) string {
	switch len(parts) {
	case 0:
		return ""
	case 1:
		return parts[0]
	}
	return strings.Join(parts[:len(parts)-1], ", ") + " and " + parts[len(parts)-1]
}

func orUnknown(s string) string {
	if s == "" {
		return "no driver"
	}
	return s
}