mesh-catalog/modules/disk-load/cmd/disk-load/measure.go

package main

// How an item is measured (novox/hq ADR 0233 §22 to §25), moved here from the backup holder, which only reads the
// readings now (ADR 0571 clause 3). Never an unbounded walk of something large.

import (
	"context"
	"fmt"
	"git.novox.be/novox/mesh-sdk/go/catalogue"
	"git.novox.be/novox/mesh-sdk/go/health"
	"regexp"
	"strconv"
	"strings"
	"time"
)

// The bounds on a walk: a walk is for small items, and one that meets a large one stops and says so.
var (
	WalkFor   = 10 * time.Minute
	WalkFiles = 2_000_000
	WalkEvery = 23 * time.Hour
)

// walkCommand sums the files under a path and finds its newest change, in one walk, bounded by time and by files.
// One line out: "<bytes> <epoch seconds> <files>", then the status of find.
func walkCommand(path string) string {
	return fmt.Sprintf("timeout %d find '%s' -xdev -printf '%%y %%s %%T@\\n' 2>/dev/null | head -n %d | "+
		"awk 'BEGIN{s=0;m=0;n=0} {n++; if ($1==\"f\") s+=$2; if ($3>m) m=$3} END {printf \"%%d %%.0f %%d\\n\", s, m, n}'; "+
		"echo \"${PIPESTATUS[0]:-0}\"", int(WalkFor.Seconds()), path, WalkFiles)
}

// topCommand is the newest change among a path and its top-level entries, and how many there are.
func topCommand(path string) string {
	return "timeout 60 find '" + path + "' -maxdepth 1 -printf '%T@\\n' 2>/dev/null | " +
		"awk 'BEGIN{m=0;n=0} {n++; if ($1>m) m=$1} END {printf \"%.0f %d\\n\", m, n-1}'"
}

func epochTime(s string) *time.Time {
	epoch, err := strconv.ParseInt(s, 10, 64)
	if err != nil || epoch <= 0 {
		return nil
	}
	t := time.Unix(epoch, 0).UTC()
	return &t
}

// Measurer measures items through a runner that has the root the walks need.
type Measurer struct {
	Run Runner
	Now func() time.Time
}

func (b Measurer) exists(ctx context.Context, path string) bool {
	_, err := b.Run(ctx, "test", "-e", path)
	return err == nil
}

// Measure is one item, measured now, by its own method.
func (b Measurer) Measure(ctx context.Context, it catalogue.PlacedDataItem) health.Measured {
	at := b.Now().UTC()
	m := health.Measured{MeasuredAt: &at}
	if !b.exists(ctx, it.Path) {
		m.Error = it.Path + " does not exist"
		zero := int64(0)
		m.SizeBytes, m.Precision = &zero, "exact"
		return m
	}
	m.Redundancy = b.redundancyOf(ctx, it.Path)
	switch it.Measure {
	case catalogue.MeasureDataset, catalogue.MeasureShallow:
		out, err := b.Run(ctx, "bash", "-c", topCommand(it.Path))
		if f := strings.Fields(out); err == nil && len(f) == 2 {
			m.LastWrite = epochTime(f[0])
		} else if err != nil {
			m.Error = err.Error()
		}
		m.Precision = "no size: the newest change among its top-level entries only"
		if it.Measure == catalogue.MeasureShallow {
			return m
		}
		out, err = b.Run(ctx, "zfs", "list", "-Hp", "-o", "name,used", it.Path)
		f := strings.Fields(out)
		if err != nil || len(f) != 2 {
			m.Precision = "no size: it is on no ZFS dataset to read one from; the newest change among its top-level entries only"
			return m
		}
		used, err := strconv.ParseInt(f[1], 10, 64)
		if err != nil {
			return m
		}
		m.SizeBytes = &used
		m.Precision = "dataset " + f[0] + ": its whole size, from the filesystem's counters; the newest change among the top-level entries"
		return m
	}
	out, err := b.Run(ctx, "bash", "-c", walkCommand(it.Path))
	if err != nil {
		m.Error = err.Error()
		return m
	}
	lines := strings.Split(strings.TrimSpace(out), "\n")
	f := strings.Fields(lines[0])
	if len(f) != 3 || len(lines) < 2 {
		m.Error = "the measurement said " + strings.TrimSpace(out)
		return m
	}
	size, err1 := strconv.ParseInt(f[0], 10, 64)
	files, err2 := strconv.Atoi(f[2])
	if err1 != nil || err2 != nil {
		m.Error = "the measurement said " + strings.TrimSpace(out)
		return m
	}
	m.SizeBytes, m.LastWrite = &size, epochTime(f[1])
	m.Precision = "exact"
	if status := strings.TrimSpace(lines[len(lines)-1]); status == "124" || files >= WalkFiles {
		m.Precision = fmt.Sprintf("partial: measured partially — stopped after %s or %d files; the size is a lower bound, "+
			"and this item wants measure: dataset or shallow", WalkFor, WalkFiles)
	}
	return m
}

// redundancyOf is the redundant storage a path is on: a ZFS pool's health, an md array's members, a btrfs
// filesystem's error counters. Nil for storage with no redundancy it knows how to read.
func (b Measurer) redundancyOf(ctx context.Context, path string) *health.Redundancy {
	out, err := b.Run(ctx, "findmnt", "-no", "SOURCE,FSTYPE", "--target", path)
	if err != nil {
		return nil
	}
	f := strings.Fields(out)
	if len(f) < 2 {
		return nil
	}
	source, fstype := f[0], f[1]
	yes, no := true, false
	switch {
	case fstype == "zfs":
		pool, _, _ := strings.Cut(source, "/")
		r := &health.Redundancy{Kind: "zfs", Where: pool}
		health, err := b.Run(ctx, "zpool", "list", "-H", "-o", "health", pool)
		if err != nil {
			r.Said = "zpool list: " + err.Error()
			return r
		}
		status, err := b.Run(ctx, "zpool", "status", "-x", pool)
		if err != nil {
			r.Said = "zpool status: " + err.Error()
			return r
		}
		health, status = strings.TrimSpace(health), strings.TrimSpace(status)
		r.Said = "health " + health + "; " + firstLineOf(status)
		if health == "ONLINE" && strings.Contains(status, "is healthy") {
			r.Healthy = &yes
		} else {
			r.Healthy = &no
			r.Said = "health " + health + "; " + oneLineOf(status)
		}
		return r
	case strings.HasPrefix(source, "/dev/md"):
		device := strings.TrimPrefix(source, "/dev/")
		r := &health.Redundancy{Kind: "md", Where: device}
		mdstat, err := b.Run(ctx, "cat", "/proc/mdstat")
		if err != nil {
			r.Said = err.Error()
			return r
		}
		healthy, said, found := mdHealth(mdstat, device)
		if !found {
			r.Said = device + " is not in /proc/mdstat"
			return r
		}
		r.Said = said
		if healthy {
			r.Healthy = &yes
		} else {
			r.Healthy = &no
		}
		return r
	case fstype == "btrfs":
		r := &health.Redundancy{Kind: "btrfs", Where: source}
		stats, err := b.Run(ctx, "btrfs", "device", "stats", "--check", path)
		if err != nil {
			r.Healthy, r.Said = &no, "device errors: "+oneLineOf(err.Error())
			return r
		}
		r.Healthy, r.Said = &yes, firstLineOf(stats)
		return r
	}
	return nil
}

var mdMembers = regexp.MustCompile(`\[([U_]+)\]`)

// mdHealth reads one array's block of /proc/mdstat: healthy when every member is up and it is not recovering.
func mdHealth(mdstat, device string) (bool, string, bool) {
	lines := strings.Split(mdstat, "\n")
	for i, l := range lines {
		if !strings.HasPrefix(l, device+" ") {
			continue
		}
		block := l
		for j := i + 1; j < len(lines) && strings.HasPrefix(lines[j], " "); j++ {
			block += " " + strings.TrimSpace(lines[j])
		}
		members := mdMembers.FindStringSubmatch(block)
		healthy := members != nil && !strings.Contains(members[1], "_") && !strings.Contains(block, "recovery")
		return healthy, oneLineOf(block), true
	}
	return false, "", false
}

func firstLineOf(s string) string {
	line, _, _ := strings.Cut(strings.TrimSpace(s), "\n")
	return line
}

func oneLineOf(s string) string { return strings.Join(strings.Fields(s), " ") }