Add disk_folders to disk-load: no tool said which folders fill a disk (issue 1804)

modules/disk-load/README.md

@@ -24,7 +24,7 @@ machine's resources a seat, these tools become its verbs.
 
 ## Tools
 
-All five only read.
+All six only read.
 
 | tool | what | replaces |
 |---|---|---|
@@ -32,6 +32,7 @@ All five only read.
 | `disk_top` | Over a window. `by=unit` (the default) ranks systemd units and containers by bytes moved, from their cgroups. `by=container` ranks containers by name, and `by=process` ranks processes | `iotop` |
 | `disk_filesystems` | Each filesystem that stores data, said once however many places it is mounted: size, used and free GiB, used % as `df` counts it, inodes used % | `df -h`, `df -i` |
 | `disk_devices` | Each disk without sampling: model, size, whether it spins, scheduler, queue limit, what it is built on, and since boot the GiB read and written, the busy share and the wait per read and per write | `lsblk`, `cat /proc/diskstats` |
+| `disk_folders` | Which folders fill a disk, like `du -x`: the largest folders under `path` (default `/`) to `depth` levels (default 2, at most 3), `limit` of them (default 20, at most 100), with bytes and GiB. Sizes are allocated bytes, a hard link once. It stays on the path's one filesystem: a mount below it is named, never entered. It reads every folder the tool runner can read (docker's included) and no content; folders it cannot read are counted and named. After 60 s it answers what it has measured, marked `partial`, with `stopped_at` (novox/hq issue 1804) | `du -x`, `ncdu` |
 | `cpu_top` | Samples a window (`seconds`, default 3, at most 10). It answers the machine's CPU in use (% of all CPUs and as CPUs busy, from `/proc/stat`), then `by=unit` (the default) ranks systemd units and containers by CPU, from their cgroups' cpu.stat, busiest first (`limit`, default 15), with the busy time no unit accounts for; `by=container` ranks containers by name | `top`, `systemd-cgtop` |
 
 ### How each number is read
@@ -87,4 +88,5 @@ cover:
 - processes ranked, one that started inside the window left out, unreadable ones counted;
 - filesystems said once, mount escapes undone, a statfs that fails said;
 - the window's bounds;
+- folders ranked biggest first to the depth asked, a mount never entered, unreadable folders named, a walk past its bound answered as partial with where it stopped, a hard link counted once;
 - the manifest: tools listed equal tools served, all read-only, no resources, no machine named.

modules/disk-load/cmd/disk-load/folders.go

@@ -0,0 +1,197 @@
+package main
+
+import (
+	"fmt"
+	"io/fs"
+	"math"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"syscall"
+	"time"
+)
+
+// FolderBudget is how long a folder walk may take before it answers what it has measured.
+const FolderBudget = 60 * time.Second
+
+// namedUnreadable is how many unreadable folders are named; the rest are only counted.
+const namedUnreadable = 50
+
+// Folder is one folder's size: everything under it on the same filesystem.
+type Folder struct {
+	Path  string  `json:"path"`
+	Depth int     `json:"depth"`
+	Bytes int64   `json:"bytes"`
+	GiB   float64 `json:"gib"`
+}
+
+// FolderReport is the answer of disk_folders.
+type FolderReport struct {
+	Path              string   `json:"path"`
+	Bytes             int64    `json:"bytes"`
+	GiB               float64  `json:"gib"`
+	Largest           []Folder `json:"largest"`
+	Partial           bool     `json:"partial"`
+	StoppedAt         string   `json:"stopped_at,omitempty"`
+	Unreadable        int      `json:"unreadable"`
+	UnreadableFolders []string `json:"unreadable_folders,omitempty"`
+	Skipped           []string `json:"other_filesystems_skipped,omitempty"`
+	Said              string   `json:"said"`
+}
+
+func (m *Machine) readDir(real string) ([]os.DirEntry, error) {
+	if m.ReadDir != nil {
+		return m.ReadDir(real)
+	}
+	return os.ReadDir(real)
+}
+
+func (m *Machine) now() time.Time {
+	if m.Clock != nil {
+		return m.Clock()
+	}
+	return time.Now()
+}
+
+func devOf(path string, info fs.FileInfo) uint64 {
+	if st, ok := info.Sys().(*syscall.Stat_t); ok {
+		return uint64(st.Dev)
+	}
+	return 0
+}
+
+type inodeKey struct{ dev, ino uint64 }
+
+// walk is one folder measurement.
+type walk struct {
+	m        *Machine
+	maxDepth int
+	dev      uint64
+	deadline time.Time
+	seen     map[inodeKey]bool
+	folders  []Folder
+	bad      []string
+	skipped  []string
+	stopped  string
+}
+
+// Folders measures path like du -x: allocated bytes, a hard link once, never entering another filesystem.
+func (m *Machine) Folders(path string, depth, limit int) (FolderReport, error) {
+	if !filepath.IsAbs(path) {
+		return FolderReport{}, fmt.Errorf("path must be absolute, not %q", path)
+	}
+	path = filepath.Clean(path)
+	info, err := os.Stat(m.path(path))
+	if err != nil {
+		return FolderReport{}, fmt.Errorf("path %s: %w", path, err)
+	}
+	if !info.IsDir() {
+		return FolderReport{}, fmt.Errorf("path %s is not a folder", path)
+	}
+	budget := m.Budget
+	if budget == 0 {
+		budget = FolderBudget
+	}
+	w := &walk{m: m, maxDepth: depth, deadline: m.now().Add(budget), seen: map[inodeKey]bool{}}
+	w.dev = w.device(path, info)
+	total := w.dir(path, 0)
+
+	sort.SliceStable(w.folders, func(i, j int) bool {
+		if w.folders[i].Bytes != w.folders[j].Bytes {
+			return w.folders[i].Bytes > w.folders[j].Bytes
+		}
+		return w.folders[i].Path < w.folders[j].Path
+	})
+	if len(w.folders) > limit {
+		w.folders = w.folders[:limit]
+	}
+	sort.Strings(w.bad)
+	named := w.bad
+	if len(named) > namedUnreadable {
+		named = named[:namedUnreadable]
+	}
+	r := FolderReport{
+		Path: path, Bytes: total, GiB: gibOf(total), Largest: w.folders,
+		Partial: w.stopped != "", StoppedAt: w.stopped,
+		Unreadable: len(w.bad), UnreadableFolders: named, Skipped: w.skipped,
+	}
+	if r.Largest == nil {
+		r.Largest = []Folder{}
+	}
+	var said []string
+	if r.Partial {
+		said = append(said, fmt.Sprintf("PARTIAL: the walk stopped after %s at %s; sizes are what was measured so far", budget, r.StoppedAt))
+	}
+	said = append(said, fmt.Sprintf("%s holds %.1f GiB on its own filesystem", path, r.GiB))
+	if r.Unreadable > 0 {
+		said = append(said, fmt.Sprintf("%d folders could not be read (%s), so their size is missing", r.Unreadable, strings.Join(named, ", ")))
+	}
+	if len(r.Skipped) > 0 {
+		said = append(said, "other filesystems not entered: "+strings.Join(r.Skipped, ", "))
+	}
+	r.Said = strings.Join(said, "; ")
+	return r, nil
+}
+
+func (w *walk) device(path string, info fs.FileInfo) uint64 {
+	if w.m.DevOf != nil {
+		return w.m.DevOf(path, info)
+	}
+	return devOf(path, info)
+}
+
+// dir is the allocated bytes under a folder; it records the folder when it is within the depth.
+func (w *walk) dir(path string, depth int) int64 {
+	if w.stopped != "" {
+		return 0
+	}
+	if w.m.now().After(w.deadline) {
+		w.stopped = path
+		return 0
+	}
+	entries, err := w.m.readDir(w.m.path(path))
+	if err != nil {
+		w.bad = append(w.bad, path)
+		return 0
+	}
+	var total int64
+	for _, e := range entries {
+		child := filepath.Join(path, e.Name())
+		info, err := os.Lstat(w.m.path(child))
+		if err != nil {
+			continue // gone since it was listed
+		}
+		if info.IsDir() {
+			if w.device(child, info) != w.dev {
+				w.skipped = append(w.skipped, child)
+				continue
+			}
+			total += w.allocated(info, false) + w.dir(child, depth+1)
+			continue
+		}
+		total += w.allocated(info, true)
+	}
+	if depth >= 1 && depth <= w.maxDepth {
+		w.folders = append(w.folders, Folder{Path: path, Depth: depth, Bytes: total, GiB: gibOf(total)})
+	}
+	return total
+}
+
+// allocated is the bytes an entry occupies on disk, as du counts them; a file with several links once.
+func (w *walk) allocated(info fs.FileInfo, file bool) int64 {
+	st, ok := info.Sys().(*syscall.Stat_t)
+	if !ok {
+		return info.Size()
+	}
+	if file && st.Nlink > 1 {
+		k := inodeKey{uint64(st.Dev), uint64(st.Ino)}
+		if w.seen[k] {
+			return 0
+		}
+		w.seen[k] = true
+	}
+	return st.Blocks * 512
+}
+
+func gibOf(b int64) float64 { return math.Round(float64(b)/gib*100) / 100 }

modules/disk-load/cmd/disk-load/folders_test.go

@@ -0,0 +1,163 @@
+package main
+
+import (
+	"fmt"
+	"io/fs"
+	"os"
+	"strings"
+	"testing"
+	"time"
+)
+
+const mb = 1 << 20
+
+// megabytes writes a file of n MiB under the fake root.
+func (f *fake) megabytes(path string, n int) { f.file(path, strings.Repeat("x", n*mb)) }
+
+func aFullDisk(t *testing.T) *fake {
+	f := newFake(t)
+	f.megabytes("var/lib/docker/overlay2/a/layer", 8)
+	f.megabytes("var/lib/docker/overlay2/b/layer", 4)
+	f.megabytes("var/log/big.log", 3)
+	f.megabytes("home/jo/video.mkv", 6)
+	f.megabytes("home/jo/deep/er/est/file", 1)
+	f.dir("empty")
+	return f
+}
+
+func folders(t *testing.T, m *Machine, args map[string]any) FolderReport {
+	t.Helper()
+	out, err := tool(t, m, "disk_folders").Run(args)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return out.(FolderReport)
+}
+
+func paths(r FolderReport) string {
+	var p []string
+	for _, f := range r.Largest {
+		p = append(p, f.Path)
+	}
+	return strings.Join(p, ",")
+}
+
+// 1804/AC1: given a disk with folders of known sizes, when disk_folders runs with the defaults, then it answers
+// the largest folders to depth 2, biggest first, with bytes and GiB, and the total of the path.
+func TestFoldersAreRankedBiggestFirstToDepthTwoByDefault(t *testing.T) {
+	r := folders(t, aFullDisk(t).machine(), nil)
+	if got := paths(r); !strings.HasPrefix(got, "/var,/var/lib,") || !strings.Contains(got, "/home,/home/jo") {
+		t.Fatalf("ranked %s", got)
+	}
+	if r.Path != "/" || r.Partial || r.Unreadable != 0 {
+		t.Errorf("%+v", r)
+	}
+	top := r.Largest[0]
+	if top.Path != "/var" || top.Depth != 1 || top.Bytes < 15*mb || top.Bytes > 16*mb || top.GiB <= 0 {
+		t.Errorf("top %+v", top)
+	}
+	for _, f := range r.Largest {
+		if f.Depth > 2 || f.Depth < 1 {
+			t.Errorf("%+v is outside depth 2", f)
+		}
+		if strings.Contains(f.Path, "overlay2") {
+			t.Errorf("%s is deeper than 2", f.Path)
+		}
+	}
+	if r.Bytes < 22*mb || r.Bytes > 24*mb {
+		t.Errorf("total %d", r.Bytes)
+	}
+}
+
+// 1804/AC2: given a path, a depth and a limit, then only that folder is measured, depth is held at 3 and limit at 100.
+func TestFoldersTakeAPathADepthAndALimitHeldAtThreeAndAHundred(t *testing.T) {
+	f := aFullDisk(t)
+	r := folders(t, f.machine(), map[string]any{"path": "/var/lib/docker", "depth": 1.0})
+	if r.Path != "/var/lib/docker" || paths(r) != "/var/lib/docker/overlay2" {
+		t.Errorf("%s: %s", r.Path, paths(r))
+	}
+	r = folders(t, f.machine(), map[string]any{"path": "/home", "depth": 9.0, "limit": 2.0})
+	if len(r.Largest) != 2 {
+		t.Errorf("limit 2 gave %d", len(r.Largest))
+	}
+	r = folders(t, f.machine(), map[string]any{"path": "/home", "depth": 9.0})
+	if got := paths(r); !strings.Contains(got, "/home/jo/deep/er") || strings.Contains(got, "/home/jo/deep/er/est") {
+		t.Errorf("depth 3 under /home is /home/jo/deep/er: %s", got)
+	}
+	tl := tool(t, f.machine(), "disk_folders")
+	for _, bad := range []map[string]any{{"depth": 0.0}, {"limit": 0.0}, {"depth": "deep"}, {"path": "relative/dir"}, {"path": "/nope"}, {"path": "/var/log/big.log"}} {
+		if _, err := tl.Run(bad); err == nil {
+			t.Errorf("%v was accepted", bad)
+		}
+	}
+	for i := 0; i < 120; i++ {
+		f.file(fmt.Sprintf("many/d%03d/f", i), "x")
+	}
+	if r := folders(t, f.machine(), map[string]any{"path": "/many", "limit": 500.0}); len(r.Largest) != 100 {
+		t.Errorf("limit held at %d, not 100", len(r.Largest))
+	}
+}
+
+// 1804/AC3: given a mount below the path, then it is never entered or counted, and is named.
+func TestFoldersNeverCrossIntoAnotherFilesystem(t *testing.T) {
+	f := aFullDisk(t)
+	m := f.machine()
+	m.DevOf = func(path string, info fs.FileInfo) uint64 {
+		if path == "/var/lib/docker" {
+			return 7
+		}
+		return 1
+	}
+	r := folders(t, m, nil)
+	if strings.Contains(paths(r), "docker") || len(r.Skipped) != 1 || r.Skipped[0] != "/var/lib/docker" {
+		t.Errorf("%s, skipped %v", paths(r), r.Skipped)
+	}
+	if r.Bytes > 14*mb {
+		t.Errorf("total %d includes the mount", r.Bytes)
+	}
+}
+
+// 1804/AC4: given folders it cannot read, then they are counted and named and the rest is still answered.
+func TestUnreadableFoldersAreCountedAndNamed(t *testing.T) {
+	f := aFullDisk(t)
+	m := f.machine()
+	real := os.ReadDir
+	m.ReadDir = func(p string) ([]os.DirEntry, error) {
+		if strings.HasSuffix(p, "/var/log") || strings.HasSuffix(p, "/overlay2/b") {
+			return nil, fmt.Errorf("open %s: permission denied", p)
+		}
+		return real(p)
+	}
+	r := folders(t, m, map[string]any{"depth": 3.0})
+	if r.Unreadable != 2 || strings.Join(r.UnreadableFolders, ",") != "/var/lib/docker/overlay2/b,/var/log" {
+		t.Errorf("%d: %v", r.Unreadable, r.UnreadableFolders)
+	}
+	if r.Partial || !strings.Contains(r.Said, "2 folders could not be read") {
+		t.Errorf("%+v", r)
+	}
+}
+
+// 1804/AC5: given a walk that outlasts its bound, then it answers what it measured, partial, with where it stopped.
+func TestALongWalkAnswersPartialAndSaysWhereItStopped(t *testing.T) {
+	f := aFullDisk(t)
+	m := f.machine()
+	now := time.Unix(0, 0)
+	m.Clock = func() time.Time { now = now.Add(10 * time.Second); return now }
+	r := folders(t, m, nil)
+	if !r.Partial || r.StoppedAt == "" || !strings.Contains(r.Said, "PARTIAL") || !strings.Contains(r.Said, r.StoppedAt) {
+		t.Errorf("%+v", r)
+	}
+}
+
+// 1804/AC6: a hard link is counted once, and the tool changes nothing.
+func TestAHardLinkIsCountedOnce(t *testing.T) {
+	f := newFake(t)
+	f.megabytes("a/big", 5)
+	if err := os.Link(f.root+"/a/big", f.root+"/a/again"); err != nil {
+		t.Skip(err)
+	}
+	r := folders(t, f.machine(), nil)
+	if r.Bytes > 6*mb {
+		t.Errorf("counted twice: %d", r.Bytes)
+	}
+}

modules/disk-load/cmd/disk-load/machine.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io/fs"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -53,6 +54,13 @@ type Machine struct {
 	Run    Runner
 	Wait   func(time.Duration)
 	Statfs func(path string) (FSStat, error)
+
+	// The folder walk's seams, all optional: the folder reader, the filesystem a path is on, the clock and
+	// the time the walk may take (60 s when zero).
+	ReadDir func(path string) ([]os.DirEntry, error)
+	DevOf   func(path string, info fs.FileInfo) uint64
+	Clock   func() time.Time
+	Budget  time.Duration
 }
 
 // Here is the machine this process runs on.

modules/disk-load/cmd/disk-load/tools.go

@@ -136,6 +136,34 @@ func Tools(m *Machine) []stdio.Tool {
 				}, nil
 			},
 		},
+		{
+			Name: "disk_folders",
+			Description: "Which folders fill a disk, like du -x: the largest folders under a path, measured on disk (allocated bytes, a hard link " +
+				"once), staying on the path's one filesystem (a mount below it is named, never entered). path defaults to /, depth to 2 (at most 3), " +
+				"limit to 20 (at most 100). Sizes in bytes and GiB. It reads every folder the tool runner can read, none of the content; folders " +
+				"it cannot read are counted and named. It stops after 60 s and answers what it measured so far, marked partial, with where it stopped. " +
+				"Replaces du. (r)",
+			Input: map[string]any{
+				"path":  map[string]any{"type": "string", "description": "the absolute folder to measure (default /)"},
+				"depth": map[string]any{"type": "integer", "description": "how many levels below path to rank (default 2, at most 3)"},
+				"limit": map[string]any{"type": "integer", "description": "how many folders to answer (default 20, at most 100)"},
+			},
+			Run: func(args map[string]any) (any, error) {
+				depth, err := count(args, "depth", 2, 1, 3)
+				if err != nil {
+					return nil, err
+				}
+				limit, err := count(args, "limit", 20, 1, 100)
+				if err != nil {
+					return nil, err
+				}
+				path := text(args, "path")
+				if path == "" {
+					path = "/"
+				}
+				return m.Folders(path, depth, limit)
+			},
+		},
 		{
 			Name: "disk_filesystems",
 			Description: "The space of every filesystem that stores data (not proc, tmpfs or overlay layers): size, used and free in GiB, " +

modules/disk-load/module.json

@@ -5,6 +5,7 @@
     "disk_load",
     "disk_top",
     "disk_filesystems",
+    "disk_folders",
     "disk_devices",
     "cpu_top"
   ],