mesh-catalog/modules/sensors/cmd/sensors/watch.go

package main

import (
	"context"
	"fmt"
	"os"
	"sync"
	"time"
)

// The watch: the module's long-running process, run by the node-engine as root (novox/hq ADR 0256). It
// reads each drive's health, so the tools, which run as the operator account, need no root; it never wakes
// a drive in standby; and it notes every FanEvery which fans are turning, so a fan that ran and stopped can
// be told from one that never runs. It writes ReadingsPath and nothing else.
//
// **The drives never hold up the fans.** In Run, the drives are read beside the looks at the fans, one
// pass at a time, each smartctl bounded by CommandTimeout; the file is written every FanEvery whatever a
// drive does, so a watch whose file grows old is a watch that is hung or gone, and the health check says
// so (StaleAfter).
//
// **A spinning drive is read at most hourly.** Reading SMART from an awake drive can restart its standby
// timer, so a drive read more often than its timer is set for never spins down. Whether the machines'
// drives are set to spin down at all is not known to the mesh (no tool reads a drive's standby timer), so
// the watch reads a rotational drive every SpinningEvery and smartd checks every drive at the same
// interval (its -i, in the module's drop-in). A drive in standby is never woken: smartctl is asked with -n.

const (
	FanEvery = 30 * time.Second
	// SolidStateEvery is how often a drive that does not spin is read.
	SolidStateEvery = 10 * time.Minute
	// SpinningEvery is how often a rotational drive is read: longer than a spin-down timer is commonly set.
	SpinningEvery = time.Hour
)

// Watch keeps the readings between looks.
type Watch struct {
	M *Machine
	// Async reads the drives beside the fans instead of before them; Run sets it.
	Async bool

	mu       sync.Mutex
	readings Readings
	reading  bool
	done     sync.WaitGroup
}

// NewWatch starts from what an earlier watch of this boot left, so a restart neither forgets the fans
// that ran nor reads every drive again at once.
func NewWatch(m *Machine) *Watch {
	w := &Watch{M: m, readings: Readings{Drives: map[string]DriveReading{}, FansRan: map[string]time.Time{}}}
	if r, why := m.ReadReadings(); why == "" && r != nil {
		w.readings = *r
		if w.readings.Drives == nil {
			w.readings.Drives = map[string]DriveReading{}
		}
		if w.readings.FansRan == nil {
			w.readings.FansRan = map[string]time.Time{}
		}
	}
	w.readings.BootID = m.bootID()
	return w
}

// due lists the drives whose next look has come: never looked at, or looked at longer ago than their
// cadence.
func (w *Watch) due(now time.Time) (due []Drive, present map[string]bool) {
	present = map[string]bool{}
	for _, d := range w.M.BlockDrives() {
		present[d.Device] = true
		every := SolidStateEvery
		if d.Rotational {
			every = SpinningEvery
		}
		last, ok := w.readings.Drives[d.Device]
		looked := last.LookedAt
		if looked.IsZero() {
			looked = last.ReadAt
		}
		if !ok || now.Sub(looked) >= every {
			due = append(due, d)
		}
	}
	return due, present
}

// Look is one look: the fans always, the drives that are due. It writes the file.
func (w *Watch) Look() error {
	now := w.M.Now()
	fans := w.M.Fans()
	w.mu.Lock()
	for _, f := range fans {
		if f.RPM != nil && *f.RPM > 0 {
			w.readings.FansRan[f.Key] = now
		}
	}
	due, present := w.due(now)
	for name := range w.readings.Drives {
		if !present[name] {
			delete(w.readings.Drives, name)
		}
	}
	start := len(due) > 0 && !w.reading
	if start {
		w.reading = true
	}
	w.mu.Unlock()
	if start {
		w.done.Add(1)
		if w.Async {
			go w.readDrives(due)
		} else {
			w.readDrives(due)
		}
	}
	return w.write()
}

// readDrives reads each due drive, one after another, and keeps each reading as it comes.
func (w *Watch) readDrives(due []Drive) {
	defer w.done.Done()
	for _, d := range due {
		w.mu.Lock()
		var before *DriveReading
		if b, ok := w.readings.Drives[d.Device]; ok && len(b.Smartctl) > 0 {
			before = &b
		}
		w.mu.Unlock()
		got := w.M.ReadDrive(d.Device, before)
		w.mu.Lock()
		w.readings.Drives[d.Device] = got
		w.mu.Unlock()
	}
	w.mu.Lock()
	w.reading = false
	w.mu.Unlock()
}

// WaitDrives waits for a pass over the drives that is under way.
func (w *Watch) WaitDrives() { w.done.Wait() }

func (w *Watch) write() error {
	w.mu.Lock()
	defer w.mu.Unlock()
	w.readings.WrittenAt = w.M.Now()
	return w.M.WriteReadings(w.readings)
}

// Run looks at once and every FanEvery until ctx ends, the drives beside the fans. A look that cannot
// write is said on standard error, which the node-engine keeps in the unit's journal, and the watch goes
// on.
func (w *Watch) Run(ctx context.Context) {
	w.Async = true
	t := time.NewTicker(FanEvery)
	defer t.Stop()
	for {
		if err := w.Look(); err != nil {
			fmt.Fprintln(os.Stderr, "sensors watch: the readings were not written:", err)
		}
		select {
		case <-ctx.Done():
			return
		case <-t.C:
		}
	}
}