jochen
66a32b196d
The review of #129 found false alarms and a silent failure: the NVMe temperature bit, old-age attributes, a fan's minimum or latched alarm, an unrelated hot sensor and a board chip's limits could raise a condition, while a hung smartctl stopped the fans' looks and an old readings file passed as healthy. Read spinning drives hourly (smartd too) so a read never keeps one from spinning down, and skip a drive that cannot say whether it sleeps.
293 lines
10 KiB
Go
293 lines
10 KiB
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
)
|
|
|
|
// Temperatures, from the kernel's hwmon chips and its thermal zones, each with the thresholds the hardware
|
|
// gives for itself (novox/hq ADR 0256 rule 2). The mesh adds no threshold of its own: a sensor that gives
|
|
// none is said without a flag.
|
|
|
|
// Flags, said beside a reading that has reached one of the hardware's own thresholds.
|
|
const (
|
|
FlagHigh = "HIGH"
|
|
FlagCritical = "CRITICAL"
|
|
)
|
|
|
|
// Temperature is one sensor's reading, in °C.
|
|
type Temperature struct {
|
|
// Source is "hwmon" or "thermal zone".
|
|
Source string `json:"source"`
|
|
// Chip is the hwmon chip's name (k10temp, nvme, amdgpu) or the zone's type (acpitz, x86_pkg_temp).
|
|
Chip string `json:"chip"`
|
|
// Kind is what it measures: cpu, gpu, drive, board, network or other.
|
|
Kind string `json:"kind"`
|
|
// Drive is the drive a drive sensor sits in, by its block device, with its model.
|
|
Drive string `json:"drive,omitempty"`
|
|
Label string `json:"label"`
|
|
C *float64 `json:"c,omitempty"`
|
|
// High is where the hardware says it runs hot: its own maximum, or a zone's hot or passive trip.
|
|
High *float64 `json:"high_c,omitempty"`
|
|
HighIs string `json:"high_is,omitempty"`
|
|
// Critical is where the hardware says it must not go: its own critical or emergency value, or a zone's
|
|
// critical trip.
|
|
Critical *float64 `json:"critical_c,omitempty"`
|
|
CriticalIs string `json:"critical_is,omitempty"`
|
|
// Alarm is the chip's own alarm bit for this sensor, where it has one and it is set.
|
|
Alarm bool `json:"alarm,omitempty"`
|
|
Flag string `json:"flag,omitempty"`
|
|
NotRead string `json:"not_read,omitempty"`
|
|
// Where is the directory it was read from, for whoever looks closer.
|
|
Where string `json:"where"`
|
|
}
|
|
|
|
// DrivesHealth says whether this sensor's critical value makes the module unhealthy. A board chip's
|
|
// (a Super I/O's) is not to be trusted: its inputs and their limits are whatever the firmware left, and a
|
|
// wrongly wired input reads far past them on a healthy machine. It is shown, and raises nothing.
|
|
func (t Temperature) DrivesHealth() bool { return !(t.Source == "hwmon" && t.Kind == "board") }
|
|
|
|
// Name is how a person reads which sensor this is.
|
|
func (t Temperature) Name() string {
|
|
what := t.Chip
|
|
if t.Label != "" && t.Label != t.Chip {
|
|
what += " " + t.Label
|
|
}
|
|
switch {
|
|
case t.Drive != "":
|
|
return fmt.Sprintf("the drive %s, sensor %s", t.Drive, t.Label)
|
|
case t.Kind == "cpu":
|
|
return fmt.Sprintf("the CPU (%s)", what)
|
|
case t.Kind == "gpu":
|
|
return fmt.Sprintf("the GPU (%s)", what)
|
|
case t.Kind == "board":
|
|
return fmt.Sprintf("the board (%s)", what)
|
|
case t.Kind == "network":
|
|
return fmt.Sprintf("the network adapter (%s)", what)
|
|
}
|
|
return "the sensor " + what
|
|
}
|
|
|
|
// sane says whether a value in °C is a temperature or a threshold the hardware means as one. A
|
|
// threshold given as the format's largest value (an NVMe sensor's 65261.85 °C) or as 0 means none, and
|
|
// so does anything a machine cannot be at and still answer.
|
|
func sane(c float64) bool { return reading(c) && c != 0 }
|
|
|
|
// reading says whether a value in °C is a temperature a machine can be at. 0 °C is one. 127 and 127.5 °C
|
|
// are not: they are what a board chip's input with nothing on it answers (a phantom AUXTIN, the largest
|
|
// value of its eight-bit register), never a real sensor's reading.
|
|
func reading(c float64) bool { return c > -40 && c < 200 && c != 127 && c != 127.5 }
|
|
|
|
func milli(v int64) *float64 {
|
|
c := round1(float64(v) / 1000)
|
|
return &c
|
|
}
|
|
|
|
// chipKinds is what a hwmon chip or a thermal zone's type measures, by the driver's name.
|
|
var chipKinds = []struct {
|
|
prefix, kind string
|
|
}{
|
|
{"k10temp", "cpu"}, {"coretemp", "cpu"}, {"zenpower", "cpu"}, {"cpu_thermal", "cpu"}, {"cpu-thermal", "cpu"},
|
|
{"x86_pkg_temp", "cpu"}, {"via_cputemp", "cpu"}, {"soc_thermal", "cpu"}, {"TCPU", "cpu"},
|
|
{"amdgpu", "gpu"}, {"radeon", "gpu"}, {"nouveau", "gpu"}, {"i915", "gpu"}, {"xe", "gpu"}, {"nvidia", "gpu"},
|
|
{"gpu", "gpu"},
|
|
{"nvme", "drive"}, {"drivetemp", "drive"},
|
|
{"iwlwifi", "network"}, {"mt79", "network"}, {"mt76", "network"}, {"ath", "network"}, {"r8169", "network"},
|
|
{"igb", "network"}, {"ixgbe", "network"}, {"mlx", "network"}, {"bnxt", "network"},
|
|
{"acpitz", "board"}, {"pch_", "board"}, {"nct", "board"}, {"it87", "board"}, {"it86", "board"},
|
|
{"w83", "board"}, {"f71", "board"}, {"asus", "board"}, {"dell_smm", "board"}, {"thinkpad", "board"},
|
|
{"applesmc", "board"}, {"INT3400", "board"}, {"SEN", "board"}, {"TSKN", "board"},
|
|
}
|
|
|
|
func kindOf(chip string) string {
|
|
for _, k := range chipKinds {
|
|
if strings.HasPrefix(chip, k.prefix) {
|
|
return k.kind
|
|
}
|
|
}
|
|
return "other"
|
|
}
|
|
|
|
var kindOrder = map[string]int{"cpu": 0, "gpu": 1, "drive": 2, "board": 3, "network": 4, "other": 5}
|
|
|
|
// Temperatures is every temperature the machine reports: each thermal zone, then each hwmon sensor that is
|
|
// not a zone's copy (the zone has the same reading and its trip points).
|
|
func (m *Machine) Temperatures() []Temperature {
|
|
var out []Temperature
|
|
inZone := map[string]bool{}
|
|
for _, zone := range m.numbered("/sys/class/thermal", "thermal_zone") {
|
|
dir := "/sys/class/thermal/" + zone
|
|
for _, n := range m.names(dir) {
|
|
if strings.HasPrefix(n, "hwmon") {
|
|
inZone[n] = true
|
|
}
|
|
}
|
|
out = append(out, m.zone(dir))
|
|
}
|
|
drives := m.driveOfHwmon()
|
|
for _, h := range m.numbered("/sys/class/hwmon", "hwmon") {
|
|
if inZone[h] {
|
|
continue
|
|
}
|
|
dir := "/sys/class/hwmon/" + h
|
|
chip := m.read(dir + "/name")
|
|
for _, input := range m.inputs(dir, "temp") {
|
|
out = append(out, m.hwmonTemperature(dir, chip, input, drives[h]))
|
|
}
|
|
}
|
|
sort.SliceStable(out, func(i, j int) bool {
|
|
if kindOrder[out[i].Kind] != kindOrder[out[j].Kind] {
|
|
return kindOrder[out[i].Kind] < kindOrder[out[j].Kind]
|
|
}
|
|
return false
|
|
})
|
|
return out
|
|
}
|
|
|
|
// inputs lists a hwmon directory's <kind>N_input files by their stem (temp1, temp2), in numeric order.
|
|
func (m *Machine) inputs(dir, kind string) []string {
|
|
var stems []string
|
|
for _, n := range m.names(dir) {
|
|
if strings.HasPrefix(n, kind) && strings.HasSuffix(n, "_input") {
|
|
stems = append(stems, strings.TrimSuffix(n, "_input"))
|
|
}
|
|
}
|
|
sortNumbered(stems, kind)
|
|
return stems
|
|
}
|
|
|
|
func (m *Machine) hwmonTemperature(dir, chip, stem, drive string) Temperature {
|
|
t := Temperature{Source: "hwmon", Chip: chip, Kind: kindOf(chip), Drive: drive, Label: m.read(dir + "/" + stem + "_label"),
|
|
Where: dir + "/" + stem}
|
|
if t.Label == "" {
|
|
t.Label = stem
|
|
}
|
|
if drive != "" {
|
|
t.Kind = "drive"
|
|
}
|
|
v, ok := m.readInt(dir + "/" + stem + "_input")
|
|
if !ok || !reading(float64(v)/1000) {
|
|
t.NotRead = "the sensor gave no temperature"
|
|
if ok {
|
|
t.NotRead = fmt.Sprintf("the sensor answered %.1f °C, which is no temperature", float64(v)/1000)
|
|
}
|
|
return t
|
|
}
|
|
t.C = milli(v)
|
|
if h, ok := m.readInt(dir + "/" + stem + "_max"); ok && sane(float64(h)/1000) {
|
|
t.High, t.HighIs = milli(h), "its own maximum"
|
|
}
|
|
if c, ok := m.readInt(dir + "/" + stem + "_crit"); ok && sane(float64(c)/1000) {
|
|
t.Critical, t.CriticalIs = milli(c), "its own critical value"
|
|
} else if e, ok := m.readInt(dir + "/" + stem + "_emergency"); ok && sane(float64(e)/1000) {
|
|
t.Critical, t.CriticalIs = milli(e), "its own emergency value"
|
|
}
|
|
for _, alarm := range []string{"_alarm", "_crit_alarm", "_max_alarm", "_emergency_alarm"} {
|
|
if a, ok := m.readInt(dir + "/" + stem + alarm); ok && a != 0 {
|
|
t.Alarm = true
|
|
}
|
|
}
|
|
t.Flag = flagOf(t)
|
|
return t
|
|
}
|
|
|
|
// zone reads one thermal zone and its trip points: critical is the critical trip, high the hot trip or,
|
|
// without one, the lowest passive trip (where the kernel starts to slow the machine down). Active trips
|
|
// are where a fan steps up, and are nothing to report.
|
|
func (m *Machine) zone(dir string) Temperature {
|
|
typ := m.read(dir + "/type")
|
|
t := Temperature{Source: "thermal zone", Chip: typ, Kind: kindOf(typ), Label: strings.TrimPrefix(dir, "/sys/class/thermal/"),
|
|
Where: dir}
|
|
v, ok := m.readInt(dir + "/temp")
|
|
if !ok || !reading(float64(v)/1000) {
|
|
t.NotRead = "the zone gave no temperature"
|
|
if ok {
|
|
t.NotRead = fmt.Sprintf("the zone answered %.1f °C, which is no temperature", float64(v)/1000)
|
|
}
|
|
return t
|
|
}
|
|
t.C = milli(v)
|
|
var hot, passive *float64
|
|
for _, n := range m.names(dir) {
|
|
stem, ok := strings.CutSuffix(n, "_type")
|
|
if !ok || !strings.HasPrefix(stem, "trip_point_") {
|
|
continue
|
|
}
|
|
tv, ok := m.readInt(dir + "/" + stem + "_temp")
|
|
if !ok || !sane(float64(tv)/1000) {
|
|
continue
|
|
}
|
|
c := milli(tv)
|
|
switch m.read(dir + "/" + n) {
|
|
case "critical":
|
|
if t.Critical == nil || *c < *t.Critical {
|
|
t.Critical, t.CriticalIs = c, "the zone's critical trip"
|
|
}
|
|
case "hot":
|
|
if hot == nil || *c < *hot {
|
|
hot = c
|
|
}
|
|
case "passive":
|
|
if passive == nil || *c < *passive {
|
|
passive = c
|
|
}
|
|
}
|
|
}
|
|
switch {
|
|
case hot != nil:
|
|
t.High, t.HighIs = hot, "the zone's hot trip"
|
|
case passive != nil:
|
|
t.High, t.HighIs = passive, "the zone's passive trip, where the kernel slows the machine down"
|
|
}
|
|
t.Flag = flagOf(t)
|
|
return t
|
|
}
|
|
|
|
func flagOf(t Temperature) string {
|
|
if t.C == nil {
|
|
return ""
|
|
}
|
|
switch {
|
|
case t.Critical != nil && *t.C >= *t.Critical:
|
|
return FlagCritical
|
|
case t.High != nil && *t.C >= *t.High:
|
|
return FlagHigh
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// driveOfHwmon names the drive each drive sensor's hwmon chip sits in: an NVMe controller lists its hwmon
|
|
// chip and its namespaces, and a SATA drive read by drivetemp lists its chip under its device.
|
|
func (m *Machine) driveOfHwmon() map[string]string {
|
|
out := map[string]string{}
|
|
for _, ctrl := range m.numbered("/sys/class/nvme", "nvme") {
|
|
dir := "/sys/class/nvme/" + ctrl
|
|
name := ctrl
|
|
for _, n := range m.names(dir) {
|
|
if strings.HasPrefix(n, ctrl+"n") && !strings.Contains(n, "p") {
|
|
name = n
|
|
break
|
|
}
|
|
}
|
|
if model := m.read(dir + "/model"); model != "" {
|
|
name += " (" + model + ")"
|
|
}
|
|
for _, n := range m.names(dir) {
|
|
if strings.HasPrefix(n, "hwmon") {
|
|
out[n] = name
|
|
}
|
|
}
|
|
}
|
|
for _, b := range m.names("/sys/block") {
|
|
for _, n := range m.names("/sys/block/" + b + "/device/hwmon") {
|
|
name := b
|
|
if model := m.read("/sys/block/" + b + "/device/model"); model != "" {
|
|
name += " (" + model + ")"
|
|
}
|
|
out[n] = name
|
|
}
|
|
}
|
|
return out
|
|
}
|