Add disk_folders to disk-load: no tool said which folders fill a disk (issue 1804)

modules/disk-load/cmd/disk-load/folders.go

@@ -0,0 +1,197 @@
+package main
+
+import (
+	"fmt"
+	"io/fs"
+	"math"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"syscall"
+	"time"
+)
+
+// FolderBudget is how long a folder walk may take before it answers what it has measured.
+const FolderBudget = 60 * time.Second
+
+// namedUnreadable is how many unreadable folders are named; the rest are only counted.
+const namedUnreadable = 50
+
+// Folder is one folder's size: everything under it on the same filesystem.
+type Folder struct {
+	Path  string  `json:"path"`
+	Depth int     `json:"depth"`
+	Bytes int64   `json:"bytes"`
+	GiB   float64 `json:"gib"`
+}
+
+// FolderReport is the answer of disk_folders.
+type FolderReport struct {
+	Path              string   `json:"path"`
+	Bytes             int64    `json:"bytes"`
+	GiB               float64  `json:"gib"`
+	Largest           []Folder `json:"largest"`
+	Partial           bool     `json:"partial"`
+	StoppedAt         string   `json:"stopped_at,omitempty"`
+	Unreadable        int      `json:"unreadable"`
+	UnreadableFolders []string `json:"unreadable_folders,omitempty"`
+	Skipped           []string `json:"other_filesystems_skipped,omitempty"`
+	Said              string   `json:"said"`
+}
+
+func (m *Machine) readDir(real string) ([]os.DirEntry, error) {
+	if m.ReadDir != nil {
+		return m.ReadDir(real)
+	}
+	return os.ReadDir(real)
+}
+
+func (m *Machine) now() time.Time {
+	if m.Clock != nil {
+		return m.Clock()
+	}
+	return time.Now()
+}
+
+func devOf(path string, info fs.FileInfo) uint64 {
+	if st, ok := info.Sys().(*syscall.Stat_t); ok {
+		return uint64(st.Dev)
+	}
+	return 0
+}
+
+type inodeKey struct{ dev, ino uint64 }
+
+// walk is one folder measurement.
+type walk struct {
+	m        *Machine
+	maxDepth int
+	dev      uint64
+	deadline time.Time
+	seen     map[inodeKey]bool
+	folders  []Folder
+	bad      []string
+	skipped  []string
+	stopped  string
+}
+
+// Folders measures path like du -x: allocated bytes, a hard link once, never entering another filesystem.
+func (m *Machine) Folders(path string, depth, limit int) (FolderReport, error) {
+	if !filepath.IsAbs(path) {
+		return FolderReport{}, fmt.Errorf("path must be absolute, not %q", path)
+	}
+	path = filepath.Clean(path)
+	info, err := os.Stat(m.path(path))
+	if err != nil {
+		return FolderReport{}, fmt.Errorf("path %s: %w", path, err)
+	}
+	if !info.IsDir() {
+		return FolderReport{}, fmt.Errorf("path %s is not a folder", path)
+	}
+	budget := m.Budget
+	if budget == 0 {
+		budget = FolderBudget
+	}
+	w := &walk{m: m, maxDepth: depth, deadline: m.now().Add(budget), seen: map[inodeKey]bool{}}
+	w.dev = w.device(path, info)
+	total := w.dir(path, 0)
+
+	sort.SliceStable(w.folders, func(i, j int) bool {
+		if w.folders[i].Bytes != w.folders[j].Bytes {
+			return w.folders[i].Bytes > w.folders[j].Bytes
+		}
+		return w.folders[i].Path < w.folders[j].Path
+	})
+	if len(w.folders) > limit {
+		w.folders = w.folders[:limit]
+	}
+	sort.Strings(w.bad)
+	named := w.bad
+	if len(named) > namedUnreadable {
+		named = named[:namedUnreadable]
+	}
+	r := FolderReport{
+		Path: path, Bytes: total, GiB: gibOf(total), Largest: w.folders,
+		Partial: w.stopped != "", StoppedAt: w.stopped,
+		Unreadable: len(w.bad), UnreadableFolders: named, Skipped: w.skipped,
+	}
+	if r.Largest == nil {
+		r.Largest = []Folder{}
+	}
+	var said []string
+	if r.Partial {
+		said = append(said, fmt.Sprintf("PARTIAL: the walk stopped after %s at %s; sizes are what was measured so far", budget, r.StoppedAt))
+	}
+	said = append(said, fmt.Sprintf("%s holds %.1f GiB on its own filesystem", path, r.GiB))
+	if r.Unreadable > 0 {
+		said = append(said, fmt.Sprintf("%d folders could not be read (%s), so their size is missing", r.Unreadable, strings.Join(named, ", ")))
+	}
+	if len(r.Skipped) > 0 {
+		said = append(said, "other filesystems not entered: "+strings.Join(r.Skipped, ", "))
+	}
+	r.Said = strings.Join(said, "; ")
+	return r, nil
+}
+
+func (w *walk) device(path string, info fs.FileInfo) uint64 {
+	if w.m.DevOf != nil {
+		return w.m.DevOf(path, info)
+	}
+	return devOf(path, info)
+}
+
+// dir is the allocated bytes under a folder; it records the folder when it is within the depth.
+func (w *walk) dir(path string, depth int) int64 {
+	if w.stopped != "" {
+		return 0
+	}
+	if w.m.now().After(w.deadline) {
+		w.stopped = path
+		return 0
+	}
+	entries, err := w.m.readDir(w.m.path(path))
+	if err != nil {
+		w.bad = append(w.bad, path)
+		return 0
+	}
+	var total int64
+	for _, e := range entries {
+		child := filepath.Join(path, e.Name())
+		info, err := os.Lstat(w.m.path(child))
+		if err != nil {
+			continue // gone since it was listed
+		}
+		if info.IsDir() {
+			if w.device(child, info) != w.dev {
+				w.skipped = append(w.skipped, child)
+				continue
+			}
+			total += w.allocated(info, false) + w.dir(child, depth+1)
+			continue
+		}
+		total += w.allocated(info, true)
+	}
+	if depth >= 1 && depth <= w.maxDepth {
+		w.folders = append(w.folders, Folder{Path: path, Depth: depth, Bytes: total, GiB: gibOf(total)})
+	}
+	return total
+}
+
+// allocated is the bytes an entry occupies on disk, as du counts them; a file with several links once.
+func (w *walk) allocated(info fs.FileInfo, file bool) int64 {
+	st, ok := info.Sys().(*syscall.Stat_t)
+	if !ok {
+		return info.Size()
+	}
+	if file && st.Nlink > 1 {
+		k := inodeKey{uint64(st.Dev), uint64(st.Ino)}
+		if w.seen[k] {
+			return 0
+		}
+		w.seen[k] = true
+	}
+	return st.Blocks * 512
+}
+
+func gibOf(b int64) float64 { return math.Round(float64(b)/gib*100) / 100 }