mesh-catalog/modules/sensors/cmd/sensors/faults.go

package main

import (
	"strings"
)

// The kernel's own counts of hardware errors since the machine started, where the hardware reports them:
// memory errors from EDAC (a server with ECC memory has them, a laptop has none), PCIe errors from AER,
// and how often an Intel CPU was slowed for heat. Each is a few small files; a machine without them has
// the class not present.

// MemoryController is one EDAC memory controller's error counts.
type MemoryController struct {
	Controller    string `json:"controller"`
	Name          string `json:"name,omitempty"`
	Corrected     int64  `json:"corrected"`
	Uncorrectable int64  `json:"uncorrectable"`
	// Modules names each memory module that counted an error, by its label, with its counts.
	Modules []MemoryModule `json:"modules,omitempty"`
	Flag    string         `json:"flag,omitempty"`
}

// MemoryModule is one DIMM's counts, where the controller says them per module.
type MemoryModule struct {
	Label         string `json:"label"`
	Corrected     int64  `json:"corrected"`
	Uncorrectable int64  `json:"uncorrectable"`
}

// PCIError is one PCIe device that counted an error.
type PCIError struct {
	Device      string `json:"device"`
	Driver      string `json:"driver,omitempty"`
	Correctable int64  `json:"correctable"`
	NonFatal    int64  `json:"non_fatal"`
	Fatal       int64  `json:"fatal"`
	Flag        string `json:"flag,omitempty"`
}

// Faults is everything the kernel counted.
type Faults struct {
	Memory []MemoryController `json:"memory"`
	// MemoryNotPresent says why no memory controller is listed.
	MemoryNotPresent string `json:"memory_not_present,omitempty"`
	// PCIDevicesCounting is how many PCIe devices report their errors; only those that counted one are
	// listed.
	PCIDevicesCounting int        `json:"pci_devices_counting"`
	PCI                []PCIError `json:"pci"`
	// Throttled is how often the CPU was slowed for heat since the machine started, summed over its
	// cores, where the CPU counts it.
	ThrottledCore    *int64 `json:"cpu_core_throttled,omitempty"`
	ThrottledPackage *int64 `json:"cpu_package_throttled,omitempty"`
}

// Faults reads the kernel's counts of hardware errors.
func (m *Machine) Faults() Faults {
	f := Faults{Memory: []MemoryController{}, PCI: []PCIError{}}
	base := "/sys/devices/system/edac/mc"
	for _, mc := range m.numbered(base, "mc") {
		dir := base + "/" + mc
		c := MemoryController{Controller: mc, Name: m.read(dir + "/mc_name")}
		c.Corrected, _ = m.readInt(dir + "/ce_count")
		c.Uncorrectable, _ = m.readInt(dir + "/ue_count")
		for _, n := range m.names(dir) {
			if !strings.HasPrefix(n, "dimm") && !strings.HasPrefix(n, "rank") {
				continue
			}
			ce, _ := m.readInt(dir + "/" + n + "/dimm_ce_count")
			ue, _ := m.readInt(dir + "/" + n + "/dimm_ue_count")
			if ce+ue == 0 {
				continue
			}
			label := m.read(dir + "/" + n + "/dimm_label")
			if label == "" {
				label = n
			}
			c.Modules = append(c.Modules, MemoryModule{Label: label, Corrected: ce, Uncorrectable: ue})
		}
		switch {
		case c.Uncorrectable > 0:
			c.Flag = FlagCritical
		case c.Corrected > 0:
			c.Flag = FlagHigh
		}
		f.Memory = append(f.Memory, c)
	}
	if len(f.Memory) == 0 {
		f.MemoryNotPresent = "no memory controller reports its errors here (no EDAC driver, or memory without ECC)"
	}
	for _, dev := range m.names("/sys/bus/pci/devices") {
		dir := "/sys/bus/pci/devices/" + dev
		if !m.exists(dir + "/aer_dev_correctable") {
			continue
		}
		f.PCIDevicesCounting++
		e := PCIError{Device: dev, Driver: uevent(m.read(dir+"/uevent"), "DRIVER"),
			Correctable: aerTotal(m.read(dir + "/aer_dev_correctable")),
			NonFatal:    aerTotal(m.read(dir + "/aer_dev_nonfatal")),
			Fatal:       aerTotal(m.read(dir + "/aer_dev_fatal"))}
		if e.Correctable+e.NonFatal+e.Fatal == 0 {
			continue
		}
		if e.Fatal+e.NonFatal > 0 {
			e.Flag = FlagHigh
		}
		f.PCI = append(f.PCI, e)
	}
	var core, pkg int64
	counted := false
	for _, cpu := range m.numbered("/sys/devices/system/cpu", "cpu") {
		dir := "/sys/devices/system/cpu/" + cpu + "/thermal_throttle"
		if c, ok := m.readInt(dir + "/core_throttle_count"); ok {
			core += c
			counted = true
		}
		if p, ok := m.readInt(dir + "/package_throttle_count"); ok && p > pkg {
			pkg = p
		}
	}
	if counted {
		f.ThrottledCore, f.ThrottledPackage = &core, &pkg
	}
	return f
}

// aerTotal is the TOTAL_ERR_ line of one AER counter file.
func aerTotal(content string) int64 {
	for _, l := range strings.Split(content, "\n") {
		fields := strings.Fields(l)
		if len(fields) == 2 && strings.HasPrefix(fields[0], "TOTAL_ERR_") {
			var n int64
			for _, r := range fields[1] {
				if r < '0' || r > '9' {
					return 0
				}
				n = n*10 + int64(r-'0')
			}
			return n
		}
	}
	return 0
}

func uevent(content, key string) string {
	for _, l := range strings.Split(content, "\n") {
		if v, ok := strings.CutPrefix(l, key+"="); ok {
			return v
		}
	}
	return ""
}