294 lines
8.6 KiB
Go
294 lines
8.6 KiB
Go
package main
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"regexp"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"syscall"
|
|
"time"
|
|
)
|
|
|
|
// The module's second long-running process (`zfs-tools events`), run by the node-engine as root.
|
|
//
|
|
// ZFS keeps its events in a queue in the kernel, in memory, from the moment its module loads: a reboot
|
|
// loses them. `zpool events -f` gives the queue from its oldest event kept, then follows. The follower
|
|
// writes each error and change of state (resilver and scrub starts and ends among them) to its journal, so
|
|
// they outlive the reboot where the journal is kept on disk. It reads the queue from its start, so the
|
|
// pool's import and whatever came while the follower was down reach the journal too; a cursor of the boot
|
|
// and the last event read, in CursorFile, keeps it from writing an event twice when it restarts. A line
|
|
// ZFS gives when its queue was full and it dropped events is written as it comes.
|
|
//
|
|
// It writes no other file and changes nothing. It is judged by its unit (novox/hq ADR 0240): after
|
|
// MaxFollowFailures follows in a row that end within FollowSettle, it exits non-zero, so the service
|
|
// manager's restart counting and the unit's state say so.
|
|
|
|
// FollowArgv follows ZFS's events from the oldest kept, with their details.
|
|
func FollowArgv() []string { return []string{"events", "-f", "-v"} }
|
|
|
|
// CursorFile is where the follower keeps the boot and the last event it read. Under /run, so a reboot,
|
|
// which empties ZFS's queue, empties it too.
|
|
const CursorFile = "/run/zfs-events/cursor"
|
|
|
|
// BootID names this boot.
|
|
const BootID = "/proc/sys/kernel/random/boot_id"
|
|
|
|
const (
|
|
// FollowRetry is how long the follower waits before following again after zpool ended.
|
|
FollowRetry = 30 * time.Second
|
|
// FollowSettle is how long a follow must last to count as having worked.
|
|
FollowSettle = time.Minute
|
|
// MaxFollowFailures in a row end the follower with an error.
|
|
MaxFollowFailures = 5
|
|
)
|
|
|
|
// Cursor is the last event read in this boot: its id and its time. ZFS numbers events from its module's
|
|
// load, so after a reload a smaller id with a later time is a new event.
|
|
type Cursor struct {
|
|
Boot string `json:"boot"`
|
|
EID uint64 `json:"eid"`
|
|
Time time.Time `json:"time"`
|
|
path string
|
|
// OnEvent hears every event not read before, after the cursor passed it.
|
|
OnEvent func(Event) `json:"-"`
|
|
}
|
|
|
|
// LoadCursor is the cursor kept for this boot; an empty one for another boot or none kept.
|
|
func LoadCursor(path, boot string) *Cursor {
|
|
c := &Cursor{Boot: boot, path: path}
|
|
b, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return c
|
|
}
|
|
var kept Cursor
|
|
if json.Unmarshal(b, &kept) != nil || kept.Boot != boot {
|
|
return c
|
|
}
|
|
kept.path = path
|
|
return &kept
|
|
}
|
|
|
|
// Save keeps the cursor, atomically.
|
|
func (c *Cursor) Save(path string) error {
|
|
if path == "" {
|
|
return nil
|
|
}
|
|
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
|
return err
|
|
}
|
|
b, _ := json.Marshal(c)
|
|
tmp := path + ".new"
|
|
if err := os.WriteFile(tmp, b, 0o600); err != nil {
|
|
return err
|
|
}
|
|
return os.Rename(tmp, path)
|
|
}
|
|
|
|
// seen says whether an event was read already.
|
|
func (c *Cursor) seen(eid uint64, at time.Time, timed bool) bool {
|
|
if c.EID == 0 && c.Time.IsZero() {
|
|
return false
|
|
}
|
|
if eid > c.EID {
|
|
return false
|
|
}
|
|
// A smaller or equal id: read, unless it is later than the cursor (ZFS's counter began again).
|
|
return !timed || !at.After(c.Time)
|
|
}
|
|
|
|
// eventTime reads an event's time as `zpool events` gives it: "Oct 8 2026 21:01:55.889900112".
|
|
func eventTime(s string) (time.Time, bool) {
|
|
t, err := time.ParseInLocation("Jan 2 2006 15:04:05.999999999", strings.Join(strings.Fields(s), " "), time.Local)
|
|
return t, err == nil
|
|
}
|
|
|
|
// EventLine is an event in one journal line.
|
|
func EventLine(e Event) string {
|
|
line := e.Time + " " + e.Class
|
|
keys := make([]string, 0, len(e.Details))
|
|
for k := range e.Details {
|
|
keys = append(keys, k)
|
|
}
|
|
sort.Strings(keys)
|
|
for _, k := range keys {
|
|
line += fmt.Sprintf(" %s=%s", k, e.Details[k])
|
|
}
|
|
return line
|
|
}
|
|
|
|
var droppedLine = regexp.MustCompile(`^dropped (\d+) events$`)
|
|
|
|
// Follow reads `zpool events -f -v` and writes each notable event the cursor has not seen in a line, and
|
|
// each line saying ZFS dropped events; the cursor moves past every event read and is saved as it goes.
|
|
func Follow(r io.Reader, out io.Writer, cur *Cursor) error {
|
|
sc := bufio.NewScanner(r)
|
|
sc.Buffer(make([]byte, 64*1024), 1024*1024)
|
|
var block []string
|
|
flush := func() {
|
|
if len(block) == 0 {
|
|
return
|
|
}
|
|
for _, e := range ParseEvents(strings.Join(block, "\n")) {
|
|
at, timed := eventTime(e.Time)
|
|
eid, _ := strconv.ParseUint(strings.TrimPrefix(e.Details["eid"], "0x"), 16, 64)
|
|
if cur.seen(eid, at, timed) {
|
|
continue
|
|
}
|
|
if Notable(e.Class) {
|
|
fmt.Fprintln(out, EventLine(e))
|
|
}
|
|
cur.EID = eid
|
|
if timed {
|
|
cur.Time = at
|
|
}
|
|
_ = cur.Save(cur.path)
|
|
if cur.OnEvent != nil {
|
|
cur.OnEvent(e)
|
|
}
|
|
}
|
|
block = block[:0]
|
|
}
|
|
for sc.Scan() {
|
|
l := sc.Text()
|
|
if m := droppedLine.FindStringSubmatch(strings.TrimSpace(l)); m != nil {
|
|
flush()
|
|
fmt.Fprintf(out, "ZFS dropped %s events before they were read: its queue was full\n", m[1])
|
|
continue
|
|
}
|
|
if strings.TrimSpace(l) == "" {
|
|
flush()
|
|
continue
|
|
}
|
|
block = append(block, l)
|
|
}
|
|
flush()
|
|
return sc.Err()
|
|
}
|
|
|
|
// FollowProcess is a running `zpool events -f`.
|
|
type FollowProcess interface {
|
|
Stdout() io.Reader
|
|
Kill() error
|
|
Wait() error
|
|
}
|
|
|
|
// FollowOnce follows until zpool ends or its output cannot be read. On a failure to read, zpool is
|
|
// killed before it is waited for: it would otherwise follow on, and the wait never end.
|
|
func FollowOnce(p FollowProcess, out io.Writer, cur *Cursor) error {
|
|
ferr := Follow(p.Stdout(), out, cur)
|
|
if ferr != nil {
|
|
_ = p.Kill()
|
|
}
|
|
werr := p.Wait()
|
|
if ferr != nil {
|
|
return fmt.Errorf("reading zpool events: %w", ferr)
|
|
}
|
|
if werr != nil {
|
|
return fmt.Errorf("zpool events -f ended: %w", werr)
|
|
}
|
|
return errors.New("zpool events -f ended")
|
|
}
|
|
|
|
// RunFollower follows again after each end, and gives up with an error after MaxFollowFailures follows in
|
|
// a row that did not last FollowSettle.
|
|
func RunFollower(ctx context.Context, out io.Writer, start func(context.Context) (FollowProcess, error), cur *Cursor,
|
|
cursorPath string, wait func(time.Duration)) error {
|
|
if cursorPath != "" {
|
|
cur.path = cursorPath
|
|
}
|
|
failures := 0
|
|
for ctx.Err() == nil {
|
|
began := time.Now()
|
|
var err error
|
|
p, serr := start(ctx)
|
|
if serr != nil {
|
|
err = serr
|
|
} else {
|
|
err = FollowOnce(p, out, cur)
|
|
}
|
|
if ctx.Err() != nil {
|
|
return nil
|
|
}
|
|
if time.Since(began) >= FollowSettle {
|
|
failures = 0
|
|
}
|
|
failures++
|
|
fmt.Fprintf(out, "following ZFS's events stopped (%v); %d of %d in a row\n", err, failures, MaxFollowFailures)
|
|
if failures >= MaxFollowFailures {
|
|
return fmt.Errorf("following ZFS's events failed %d times in a row, the last: %v", failures, err)
|
|
}
|
|
wait(FollowRetry)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// zpoolFollow is the real `zpool events -f -v`, in a process group of its own.
|
|
type zpoolFollow struct {
|
|
cmd *exec.Cmd
|
|
out io.ReadCloser
|
|
}
|
|
|
|
func (z *zpoolFollow) Stdout() io.Reader { return z.out }
|
|
func (z *zpoolFollow) Kill() error { return syscall.Kill(-z.cmd.Process.Pid, syscall.SIGKILL) }
|
|
func (z *zpoolFollow) Wait() error { return z.cmd.Wait() }
|
|
|
|
// StartZpoolFollow starts `zpool events -f -v`; the context's end kills it.
|
|
func StartZpoolFollow(ctx context.Context) (FollowProcess, error) {
|
|
cmd := exec.Command("zpool", FollowArgv()...)
|
|
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
|
cmd.WaitDelay = KillGrace
|
|
out, err := cmd.StdoutPipe()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err := cmd.Start(); err != nil {
|
|
return nil, err
|
|
}
|
|
z := &zpoolFollow{cmd: cmd, out: out}
|
|
go func() {
|
|
<-ctx.Done()
|
|
_ = z.Kill()
|
|
}()
|
|
return z, nil
|
|
}
|
|
|
|
// Follower is the events process: the cursor for this boot, then RunFollower.
|
|
func Follower(ctx context.Context, out io.Writer) error {
|
|
boot, _ := os.ReadFile(BootID)
|
|
cur := LoadCursor(CursorFile, strings.TrimSpace(string(boot)))
|
|
m := Here()
|
|
defer m.JoinBus()()
|
|
warn := func(what string, err error) {
|
|
if err != nil {
|
|
fmt.Fprintf(out, "%s: %v\n", what, err)
|
|
}
|
|
}
|
|
// Once at its own start (A-16), then on the events that end a wait.
|
|
warn("starting the apply", m.StartApply())
|
|
warn("reading the findings", m.SyncFindings(ctx))
|
|
cur.OnEvent = func(e Event) {
|
|
if EndsAWait(e.Class) {
|
|
warn("starting the apply", m.StartApply())
|
|
}
|
|
if ChangesAFinding(e.Class) {
|
|
warn("reading the findings", m.SyncFindings(ctx))
|
|
}
|
|
}
|
|
go func() { warn("following udev", m.StartUdev(ctx)) }()
|
|
return RunFollower(ctx, out, StartZpoolFollow, cur, CursorFile, func(d time.Duration) {
|
|
select {
|
|
case <-ctx.Done():
|
|
case <-time.After(d):
|
|
}
|
|
})
|
|
}
|