package main // How an item is measured (novox/hq ADR 0233 §22 to §25), moved here from the backup holder, which only reads the // readings now (ADR 0571 clause 3). Never an unbounded walk of something large. import ( "context" "fmt" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/health" "regexp" "strconv" "strings" "time" ) // The bounds on a walk: a walk is for small items, and one that meets a large one stops and says so. var ( WalkFor = 10 * time.Minute WalkFiles = 2_000_000 WalkEvery = 23 * time.Hour ) // walkCommand sums the files under a path and finds its newest change, in one walk, bounded by time and by files. // One line out: " ", then the status of find. func walkCommand(path string) string { return fmt.Sprintf("timeout %d find '%s' -xdev -printf '%%y %%s %%T@\\n' 2>/dev/null | head -n %d | "+ "awk 'BEGIN{s=0;m=0;n=0} {n++; if ($1==\"f\") s+=$2; if ($3>m) m=$3} END {printf \"%%d %%.0f %%d\\n\", s, m, n}'; "+ "echo \"${PIPESTATUS[0]:-0}\"", int(WalkFor.Seconds()), path, WalkFiles) } // topCommand is the newest change among a path and its top-level entries, and how many there are. func topCommand(path string) string { return "timeout 60 find '" + path + "' -maxdepth 1 -printf '%T@\\n' 2>/dev/null | " + "awk 'BEGIN{m=0;n=0} {n++; if ($1>m) m=$1} END {printf \"%.0f %d\\n\", m, n-1}'" } func epochTime(s string) *time.Time { epoch, err := strconv.ParseInt(s, 10, 64) if err != nil || epoch <= 0 { return nil } t := time.Unix(epoch, 0).UTC() return &t } // Measurer measures items through a runner that has the root the walks need. type Measurer struct { Run Runner Now func() time.Time } func (b Measurer) exists(ctx context.Context, path string) bool { _, err := b.Run(ctx, "test", "-e", path) return err == nil } // Measure is one item, measured now, by its own method. func (b Measurer) Measure(ctx context.Context, it catalogue.PlacedDataItem) health.Measured { at := b.Now().UTC() m := health.Measured{MeasuredAt: &at} if !b.exists(ctx, it.Path) { m.Error = it.Path + " does not exist" zero := int64(0) m.SizeBytes, m.Precision = &zero, "exact" return m } m.Redundancy = b.redundancyOf(ctx, it.Path) switch it.Measure { case catalogue.MeasureDataset, catalogue.MeasureShallow: out, err := b.Run(ctx, "bash", "-c", topCommand(it.Path)) if f := strings.Fields(out); err == nil && len(f) == 2 { m.LastWrite = epochTime(f[0]) } else if err != nil { m.Error = err.Error() } m.Precision = "no size: the newest change among its top-level entries only" if it.Measure == catalogue.MeasureShallow { return m } out, err = b.Run(ctx, "zfs", "list", "-Hp", "-o", "name,used", it.Path) f := strings.Fields(out) if err != nil || len(f) != 2 { m.Precision = "no size: it is on no ZFS dataset to read one from; the newest change among its top-level entries only" return m } used, err := strconv.ParseInt(f[1], 10, 64) if err != nil { return m } m.SizeBytes = &used m.Precision = "dataset " + f[0] + ": its whole size, from the filesystem's counters; the newest change among the top-level entries" return m } out, err := b.Run(ctx, "bash", "-c", walkCommand(it.Path)) if err != nil { m.Error = err.Error() return m } lines := strings.Split(strings.TrimSpace(out), "\n") f := strings.Fields(lines[0]) if len(f) != 3 || len(lines) < 2 { m.Error = "the measurement said " + strings.TrimSpace(out) return m } size, err1 := strconv.ParseInt(f[0], 10, 64) files, err2 := strconv.Atoi(f[2]) if err1 != nil || err2 != nil { m.Error = "the measurement said " + strings.TrimSpace(out) return m } m.SizeBytes, m.LastWrite = &size, epochTime(f[1]) m.Precision = "exact" if status := strings.TrimSpace(lines[len(lines)-1]); status == "124" || files >= WalkFiles { m.Precision = fmt.Sprintf("partial: measured partially — stopped after %s or %d files; the size is a lower bound, "+ "and this item wants measure: dataset or shallow", WalkFor, WalkFiles) } return m } // redundancyOf is the redundant storage a path is on: a ZFS pool's health, an md array's members, a btrfs // filesystem's error counters. Nil for storage with no redundancy it knows how to read. func (b Measurer) redundancyOf(ctx context.Context, path string) *health.Redundancy { out, err := b.Run(ctx, "findmnt", "-no", "SOURCE,FSTYPE", "--target", path) if err != nil { return nil } f := strings.Fields(out) if len(f) < 2 { return nil } source, fstype := f[0], f[1] yes, no := true, false switch { case fstype == "zfs": pool, _, _ := strings.Cut(source, "/") r := &health.Redundancy{Kind: "zfs", Where: pool} health, err := b.Run(ctx, "zpool", "list", "-H", "-o", "health", pool) if err != nil { r.Said = "zpool list: " + err.Error() return r } status, err := b.Run(ctx, "zpool", "status", "-x", pool) if err != nil { r.Said = "zpool status: " + err.Error() return r } health, status = strings.TrimSpace(health), strings.TrimSpace(status) r.Said = "health " + health + "; " + firstLineOf(status) if health == "ONLINE" && strings.Contains(status, "is healthy") { r.Healthy = &yes } else { r.Healthy = &no r.Said = "health " + health + "; " + oneLineOf(status) } return r case strings.HasPrefix(source, "/dev/md"): device := strings.TrimPrefix(source, "/dev/") r := &health.Redundancy{Kind: "md", Where: device} mdstat, err := b.Run(ctx, "cat", "/proc/mdstat") if err != nil { r.Said = err.Error() return r } healthy, said, found := mdHealth(mdstat, device) if !found { r.Said = device + " is not in /proc/mdstat" return r } r.Said = said if healthy { r.Healthy = &yes } else { r.Healthy = &no } return r case fstype == "btrfs": r := &health.Redundancy{Kind: "btrfs", Where: source} stats, err := b.Run(ctx, "btrfs", "device", "stats", "--check", path) if err != nil { r.Healthy, r.Said = &no, "device errors: "+oneLineOf(err.Error()) return r } r.Healthy, r.Said = &yes, firstLineOf(stats) return r } return nil } var mdMembers = regexp.MustCompile(`\[([U_]+)\]`) // mdHealth reads one array's block of /proc/mdstat: healthy when every member is up and it is not recovering. func mdHealth(mdstat, device string) (bool, string, bool) { lines := strings.Split(mdstat, "\n") for i, l := range lines { if !strings.HasPrefix(l, device+" ") { continue } block := l for j := i + 1; j < len(lines) && strings.HasPrefix(lines[j], " "); j++ { block += " " + strings.TrimSpace(lines[j]) } members := mdMembers.FindStringSubmatch(block) healthy := members != nil && !strings.Contains(members[1], "_") && !strings.Contains(block, "recovery") return healthy, oneLineOf(block), true } return false, "", false } func firstLineOf(s string) string { line, _, _ := strings.Cut(strings.TrimSpace(s), "\n") return line } func oneLineOf(s string) string { return strings.Join(strings.Fields(s), " ") }