mesh-catalog/modules/sensors/cmd/sensors/watch.go

package main

import (
	"context"
	"fmt"
	"os"
	"sync"
	"time"
)

// The module's two long-running processes, both of this binary, run by the node-engine (novox/hq ADR
// 0256). Each writes its own file every FanEvery and nothing else:
//
//   - **the fan watch** (`sensors watch`) notes which fans are turning, so a fan that ran and stopped can
//     be told from one that never runs. It is judged by the module's health tool, sensors_health;
//   - **the drive reader** (`sensors drives`), as root, reads each drive's health, so the tools, which run
//     as the operator account, need no root. It is judged by its unit.
//
// **One drive never holds up another, nor the fans.** Each drive is read in its own goroutine, one read at
// a time per drive, bounded by CommandTimeout and WaitDelay. A smartctl stuck in the kernel (a drive or a
// controller that stopped answering) cannot be ended at all: its read stays in flight, marked since when,
// and the health check says so once it is older than any read can take (ReadBound). The fans are another
// process, so they are looked at whatever the drives do.
//
// **The module is the only reader, and a spinning drive is read hourly.** Reading SMART from an awake
// drive can restart its standby timer, so a drive read more often than its timer is set for never spins
// down. Whether the machines' drives are set to spin down is not known to the mesh, so a rotational drive
// is read every SpinningEvery. No other reader runs: the module does not run smartd, whose own schedule
// would read the same drives at another phase of the hour, and whose start reads every drive.

const (
	FanEvery = 30 * time.Second
	// SolidStateEvery is how often a drive that does not spin is read.
	SolidStateEvery = 10 * time.Minute
	// SpinningEvery is how often a rotational drive is read: longer than a spin-down timer is commonly set.
	SpinningEvery = time.Hour
)

// ReadBound is the longest one drive's read can take when the drive answers or smartctl can be ended: a
// read in flight for longer is stuck, and said.
func ReadBound() time.Duration { return CommandTimeout + WaitDelay + FanEvery }

// cadence is how often a drive is read.
func cadence(d Drive) time.Duration {
	if d.Rotational {
		return SpinningEvery
	}
	return SolidStateEvery
}

// FanWatch keeps which fans ran since the machine started.
type FanWatch struct {
	M        *Machine
	readings Readings
}

// NewFanWatch starts from what an earlier fan watch of this boot left.
func NewFanWatch(m *Machine) *FanWatch {
	w := &FanWatch{M: m, readings: Readings{FansRan: map[string]time.Time{}}}
	if r, why := m.ReadReadings(FansPath); why == "" && r != nil && r.FansRan != nil {
		w.readings.FansRan = r.FansRan
	}
	w.readings.BootID = m.bootID()
	return w
}

// Look notes the fans turning now, and writes the file.
func (w *FanWatch) Look() error {
	now := w.M.Now()
	for _, f := range w.M.Fans() {
		if f.RPM != nil && *f.RPM > 0 {
			w.readings.FansRan[f.Key] = now
		}
	}
	w.readings.WrittenAt = now
	return w.M.WriteReadings(FansPath, w.readings)
}

// DriveReader keeps each drive's newest reading.
type DriveReader struct {
	M *Machine
	// Async reads each due drive in a goroutine of its own instead of before writing; Run sets it.
	Async bool

	mu       sync.Mutex
	readings Readings
	done     sync.WaitGroup
}

// NewDriveReader starts from what an earlier reader of this boot left, so a restart does not read every
// drive again at once. A read that was in flight when it stopped is not.
func NewDriveReader(m *Machine) *DriveReader {
	w := &DriveReader{M: m, readings: Readings{Drives: map[string]DriveReading{}}}
	if r, why := m.ReadReadings(DrivesPath); why == "" && r != nil && r.Drives != nil {
		w.readings.Drives = r.Drives
		for name, d := range w.readings.Drives {
			d.InFlightSince = nil
			w.readings.Drives[name] = d
		}
	}
	w.readings.BootID = m.bootID()
	return w
}

// Look starts a read of every drive that is due and not already being read, forgets drives that are gone,
// and writes the file.
func (w *DriveReader) Look() error {
	now := w.M.Now()
	var due []Drive
	present := map[string]bool{}
	w.mu.Lock()
	for _, d := range w.M.BlockDrives() {
		present[d.Device] = true
		last, ok := w.readings.Drives[d.Device]
		if ok && last.InFlightSince != nil {
			continue
		}
		if !ok || now.Sub(last.LookedAt) >= cadence(d) {
			last.InFlightSince = &now
			w.readings.Drives[d.Device] = last
			due = append(due, d)
		}
	}
	for name, d := range w.readings.Drives {
		if !present[name] && d.InFlightSince == nil {
			delete(w.readings.Drives, name)
		}
	}
	w.mu.Unlock()
	for _, d := range due {
		w.done.Add(1)
		if w.Async {
			go w.read(d)
		} else {
			w.read(d)
		}
	}
	return w.write()
}

// read reads one drive and keeps its reading.
func (w *DriveReader) read(d Drive) {
	defer w.done.Done()
	w.mu.Lock()
	var before *DriveReading
	if b, ok := w.readings.Drives[d.Device]; ok && (len(b.Smartctl) > 0 || b.Asleep) {
		b.InFlightSince = nil
		before = &b
	}
	w.mu.Unlock()
	got := w.M.ReadDrive(d, before)
	got.InFlightSince = nil
	w.mu.Lock()
	w.readings.Drives[d.Device] = got
	w.mu.Unlock()
}

// WaitReads waits for the reads under way.
func (w *DriveReader) WaitReads() { w.done.Wait() }

func (w *DriveReader) write() error {
	w.mu.Lock()
	defer w.mu.Unlock()
	w.readings.WrittenAt = w.M.Now()
	return w.M.WriteReadings(DrivesPath, w.readings)
}

// every runs look at once and every FanEvery until ctx ends. A look that cannot write is said on standard
// error, which the node-engine keeps in the unit's journal, and the process goes on.
func every(ctx context.Context, name string, look func() error) {
	t := time.NewTicker(FanEvery)
	defer t.Stop()
	for {
		if err := look(); err != nil {
			fmt.Fprintf(os.Stderr, "sensors %s: the readings were not written: %v\n", name, err)
		}
		select {
		case <-ctx.Done():
			return
		case <-t.C:
		}
	}
}