Files
mesh-catalog/modules/sensors/cmd/sensors/machine.go
T
jochen 66a32b196d
mesh/merge-gate pass: builds new: modules/sensors, sent nowhere; no bus step; every machine composes with the change as it did without (4 of 4 compose)
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery superseded: a newer head of the same pull request
sensors: raise only what the hardware says is failing, and say a hung watch
The review of #129 found false alarms and a silent failure: the NVMe temperature bit, old-age
attributes, a fan's minimum or latched alarm, an unrelated hot sensor and a board chip's limits could
raise a condition, while a hung smartctl stopped the fans' looks and an old readings file passed as
healthy. Read spinning drives hourly (smartd too) so a read never keeps one from spinning down, and skip
a drive that cannot say whether it sleeps.
2026-10-08 14:35:16 +02:00

164 lines
4.9 KiB
Go

package main
import (
"bytes"
"context"
"errors"
"fmt"
"io/fs"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"time"
)
// What the module reads, and as whom.
//
// The tools run as the operator account in the node's tool runner (novox/hq ADR 0175 §4) and read only
// /sys and /proc, which every account may. A drive's health needs root: the NVMe controller's device and a
// SATA drive's SMART commands are root's. So the module's watch, a process of this same binary that the
// node-engine runs as root, reads it with smartctl and leaves its readings in one file the tools read
// (novox/hq ADR 0256).
// CommandTimeout bounds one smartctl: a drive that does not answer costs one line, never the watch. A
// variable only so a test can shorten it.
var CommandTimeout = 20 * time.Second
// WaitDelay is how long a command ended for its timeout may keep its output open (a child it left behind)
// before the pipes are closed under it: without it, the wait for the output outlives the timeout.
const WaitDelay = 2 * time.Second
// Ran is what one command did: its output and exit status, and why it never ran to an answer.
type Ran struct {
Stdout string
Stderr string
Status int
// Err is "ENOENT" when the program is not there, or that it was ended for taking too long.
Err string
}
// Runner runs one command; injected, so the watch is tested over recorded answers.
type Runner func(ctx context.Context, name string, args ...string) Ran
// ExecRunner runs a command on this machine, in the C locale, bounded by CommandTimeout.
func ExecRunner(ctx context.Context, name string, args ...string) Ran {
ctx, cancel := context.WithTimeout(ctx, CommandTimeout)
defer cancel()
cmd := exec.CommandContext(ctx, name, args...)
cmd.Env = append(os.Environ(), "LC_ALL=C")
cmd.WaitDelay = WaitDelay
var out, errb bytes.Buffer
cmd.Stdout, cmd.Stderr = &out, &errb
err := cmd.Run()
r := Ran{Stdout: out.String(), Stderr: errb.String()}
if ctx.Err() == context.DeadlineExceeded || errors.Is(err, exec.ErrWaitDelay) {
r.Status, r.Err = 124, fmt.Sprintf("no answer within %s", CommandTimeout)
return r
}
var exit *exec.ExitError
switch {
case err == nil:
case errors.As(err, &exit):
r.Status = exit.ExitCode()
case errors.Is(err, exec.ErrNotFound) || errors.Is(err, fs.ErrNotExist):
r.Status, r.Err = 127, "ENOENT"
default:
r.Status, r.Err = 126, err.Error()
}
return r
}
// Machine is what the module reads: a filesystem root (the real one, or a test's tree standing in for
// /sys, /proc and /run), a way to run a command, and the time.
type Machine struct {
Root string
Run Runner
Now func() time.Time
// UID is who this process is: smartctl is run directly by root and through sudo -n by anyone else.
UID int
}
// Here is the machine this process runs on.
func Here() *Machine {
return &Machine{Root: "/", Run: ExecRunner, Now: time.Now, UID: os.Getuid()}
}
func (m *Machine) path(p string) string { return filepath.Join(m.Root, p) }
// read is a file's content, trimmed; "" when it cannot be read.
func (m *Machine) read(p string) string {
b, err := os.ReadFile(m.path(p))
if err != nil {
return ""
}
return strings.TrimSpace(string(b))
}
// readInt is a file holding one whole number; false when it is absent, unreadable or not a number — a
// sensor whose driver answers an error (ENODATA, EIO) is said as not read, never as 0.
func (m *Machine) readInt(p string) (int64, bool) {
s := m.read(p)
if s == "" {
return 0, false
}
n, err := strconv.ParseInt(s, 10, 64)
return n, err == nil
}
func (m *Machine) exists(p string) bool {
_, err := os.Stat(m.path(p))
return err == nil
}
// names lists the entries of a directory, sorted; none when it cannot be read.
func (m *Machine) names(dir string) []string {
entries, err := os.ReadDir(m.path(dir))
if err != nil {
return nil
}
out := make([]string, 0, len(entries))
for _, e := range entries {
out = append(out, e.Name())
}
return out
}
// numbered lists a directory's entries named prefix<number>, in numeric order (hwmon2 before hwmon10).
func (m *Machine) numbered(dir, prefix string) []string {
var out []string
for _, n := range m.names(dir) {
if rest, ok := strings.CutPrefix(n, prefix); ok {
if _, err := strconv.Atoi(rest); err == nil {
out = append(out, n)
}
}
}
sortNumbered(out, prefix)
return out
}
func sortNumbered(names []string, prefix string) {
num := func(s string) int {
n, _ := strconv.Atoi(strings.TrimPrefix(s, prefix))
return n
}
for i := 1; i < len(names); i++ {
for j := i; j > 0 && num(names[j]) < num(names[j-1]); j-- {
names[j], names[j-1] = names[j-1], names[j]
}
}
}
// bootID names this boot, so a reading from before the machine last started is not taken for this one.
func (m *Machine) bootID() string { return m.read("/proc/sys/kernel/random/boot_id") }
// round1 rounds to one decimal, for what a person reads.
func round1(f float64) float64 {
if f < 0 {
return -float64(int64(-f*10+0.5)) / 10
}
return float64(int64(f*10+0.5)) / 10
}