The review of #129 found false alarms and a silent failure: the NVMe temperature bit, old-age attributes, a fan's minimum or latched alarm, an unrelated hot sensor and a board chip's limits could raise a condition, while a hung smartctl stopped the fans' looks and an old readings file passed as healthy. Read spinning drives hourly (smartd too) so a read never keeps one from spinning down, and skip a drive that cannot say whether it sleeps.
164 lines
4.9 KiB
Go
164 lines
4.9 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io/fs"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// What the module reads, and as whom.
|
|
//
|
|
// The tools run as the operator account in the node's tool runner (novox/hq ADR 0175 §4) and read only
|
|
// /sys and /proc, which every account may. A drive's health needs root: the NVMe controller's device and a
|
|
// SATA drive's SMART commands are root's. So the module's watch, a process of this same binary that the
|
|
// node-engine runs as root, reads it with smartctl and leaves its readings in one file the tools read
|
|
// (novox/hq ADR 0256).
|
|
|
|
// CommandTimeout bounds one smartctl: a drive that does not answer costs one line, never the watch. A
|
|
// variable only so a test can shorten it.
|
|
var CommandTimeout = 20 * time.Second
|
|
|
|
// WaitDelay is how long a command ended for its timeout may keep its output open (a child it left behind)
|
|
// before the pipes are closed under it: without it, the wait for the output outlives the timeout.
|
|
const WaitDelay = 2 * time.Second
|
|
|
|
// Ran is what one command did: its output and exit status, and why it never ran to an answer.
|
|
type Ran struct {
|
|
Stdout string
|
|
Stderr string
|
|
Status int
|
|
// Err is "ENOENT" when the program is not there, or that it was ended for taking too long.
|
|
Err string
|
|
}
|
|
|
|
// Runner runs one command; injected, so the watch is tested over recorded answers.
|
|
type Runner func(ctx context.Context, name string, args ...string) Ran
|
|
|
|
// ExecRunner runs a command on this machine, in the C locale, bounded by CommandTimeout.
|
|
func ExecRunner(ctx context.Context, name string, args ...string) Ran {
|
|
ctx, cancel := context.WithTimeout(ctx, CommandTimeout)
|
|
defer cancel()
|
|
cmd := exec.CommandContext(ctx, name, args...)
|
|
cmd.Env = append(os.Environ(), "LC_ALL=C")
|
|
cmd.WaitDelay = WaitDelay
|
|
var out, errb bytes.Buffer
|
|
cmd.Stdout, cmd.Stderr = &out, &errb
|
|
err := cmd.Run()
|
|
r := Ran{Stdout: out.String(), Stderr: errb.String()}
|
|
if ctx.Err() == context.DeadlineExceeded || errors.Is(err, exec.ErrWaitDelay) {
|
|
r.Status, r.Err = 124, fmt.Sprintf("no answer within %s", CommandTimeout)
|
|
return r
|
|
}
|
|
var exit *exec.ExitError
|
|
switch {
|
|
case err == nil:
|
|
case errors.As(err, &exit):
|
|
r.Status = exit.ExitCode()
|
|
case errors.Is(err, exec.ErrNotFound) || errors.Is(err, fs.ErrNotExist):
|
|
r.Status, r.Err = 127, "ENOENT"
|
|
default:
|
|
r.Status, r.Err = 126, err.Error()
|
|
}
|
|
return r
|
|
}
|
|
|
|
// Machine is what the module reads: a filesystem root (the real one, or a test's tree standing in for
|
|
// /sys, /proc and /run), a way to run a command, and the time.
|
|
type Machine struct {
|
|
Root string
|
|
Run Runner
|
|
Now func() time.Time
|
|
// UID is who this process is: smartctl is run directly by root and through sudo -n by anyone else.
|
|
UID int
|
|
}
|
|
|
|
// Here is the machine this process runs on.
|
|
func Here() *Machine {
|
|
return &Machine{Root: "/", Run: ExecRunner, Now: time.Now, UID: os.Getuid()}
|
|
}
|
|
|
|
func (m *Machine) path(p string) string { return filepath.Join(m.Root, p) }
|
|
|
|
// read is a file's content, trimmed; "" when it cannot be read.
|
|
func (m *Machine) read(p string) string {
|
|
b, err := os.ReadFile(m.path(p))
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return strings.TrimSpace(string(b))
|
|
}
|
|
|
|
// readInt is a file holding one whole number; false when it is absent, unreadable or not a number — a
|
|
// sensor whose driver answers an error (ENODATA, EIO) is said as not read, never as 0.
|
|
func (m *Machine) readInt(p string) (int64, bool) {
|
|
s := m.read(p)
|
|
if s == "" {
|
|
return 0, false
|
|
}
|
|
n, err := strconv.ParseInt(s, 10, 64)
|
|
return n, err == nil
|
|
}
|
|
|
|
func (m *Machine) exists(p string) bool {
|
|
_, err := os.Stat(m.path(p))
|
|
return err == nil
|
|
}
|
|
|
|
// names lists the entries of a directory, sorted; none when it cannot be read.
|
|
func (m *Machine) names(dir string) []string {
|
|
entries, err := os.ReadDir(m.path(dir))
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
out := make([]string, 0, len(entries))
|
|
for _, e := range entries {
|
|
out = append(out, e.Name())
|
|
}
|
|
return out
|
|
}
|
|
|
|
// numbered lists a directory's entries named prefix<number>, in numeric order (hwmon2 before hwmon10).
|
|
func (m *Machine) numbered(dir, prefix string) []string {
|
|
var out []string
|
|
for _, n := range m.names(dir) {
|
|
if rest, ok := strings.CutPrefix(n, prefix); ok {
|
|
if _, err := strconv.Atoi(rest); err == nil {
|
|
out = append(out, n)
|
|
}
|
|
}
|
|
}
|
|
sortNumbered(out, prefix)
|
|
return out
|
|
}
|
|
|
|
func sortNumbered(names []string, prefix string) {
|
|
num := func(s string) int {
|
|
n, _ := strconv.Atoi(strings.TrimPrefix(s, prefix))
|
|
return n
|
|
}
|
|
for i := 1; i < len(names); i++ {
|
|
for j := i; j > 0 && num(names[j]) < num(names[j-1]); j-- {
|
|
names[j], names[j-1] = names[j-1], names[j]
|
|
}
|
|
}
|
|
}
|
|
|
|
// bootID names this boot, so a reading from before the machine last started is not taken for this one.
|
|
func (m *Machine) bootID() string { return m.read("/proc/sys/kernel/random/boot_id") }
|
|
|
|
// round1 rounds to one decimal, for what a person reads.
|
|
func round1(f float64) float64 {
|
|
if f < 0 {
|
|
return -float64(int64(-f*10+0.5)) / 10
|
|
}
|
|
return float64(int64(f*10+0.5)) / 10
|
|
}
|