mesh-catalog/modules/disk-load/cmd/disk-load/folders.go

package main

import (
	"fmt"
	"io/fs"
	"math"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"syscall"
	"time"
)

// FolderBudget is how long a folder walk may take before it answers what it has measured.
const FolderBudget = 25 * time.Second

// namedUnreadable is how many unreadable folders are named; the rest are only counted.
const namedUnreadable = 50

// Folder is one folder's size: everything under it on the same filesystem.
type Folder struct {
	Path  string  `json:"path"`
	Depth int     `json:"depth"`
	Bytes int64   `json:"bytes"`
	GiB   float64 `json:"gib"`
}

// FolderReport is the answer of disk_folders.
type FolderReport struct {
	Path              string   `json:"path"`
	Bytes             int64    `json:"bytes"`
	GiB               float64  `json:"gib"`
	Largest           []Folder `json:"largest"`
	Partial           bool     `json:"partial"`
	StoppedAt         string   `json:"stopped_at,omitempty"`
	Unreadable        int      `json:"unreadable"`
	UnreadableFolders []string `json:"unreadable_folders,omitempty"`
	Skipped           []string `json:"other_filesystems_skipped,omitempty"`
	Said              []string `json:"said"`
}

func (m *Machine) readDir(real string) ([]os.DirEntry, error) {
	if m.ReadDir != nil {
		return m.ReadDir(real)
	}
	return os.ReadDir(real)
}

func (m *Machine) now() time.Time {
	if m.Clock != nil {
		return m.Clock()
	}
	return time.Now()
}

func devOf(path string, info fs.FileInfo) uint64 {
	if st, ok := info.Sys().(*syscall.Stat_t); ok {
		return uint64(st.Dev)
	}
	return 0
}

type inodeKey struct{ dev, ino uint64 }

// walk is one folder measurement.
type walk struct {
	m        *Machine
	maxDepth int
	dev      uint64
	deadline time.Time
	seen     map[inodeKey]bool
	folders  []Folder
	bad      []string
	skipped  []string
	stopped  string
}

// Folders measures path like du -x: allocated bytes, a hard link once, never entering another filesystem.
func (m *Machine) Folders(path string, depth, limit int) (FolderReport, error) {
	if !filepath.IsAbs(path) {
		return FolderReport{}, fmt.Errorf("path must be absolute, not %q", path)
	}
	path = filepath.Clean(path)
	info, err := os.Stat(m.path(path))
	if err != nil {
		return FolderReport{}, fmt.Errorf("path %s: %w", path, err)
	}
	if !info.IsDir() {
		return FolderReport{}, fmt.Errorf("path %s is not a folder", path)
	}
	budget := m.Budget
	if budget == 0 {
		budget = FolderBudget
	}
	w := &walk{m: m, maxDepth: depth, deadline: m.now().Add(budget), seen: map[inodeKey]bool{}}
	w.dev = w.device(path, info)
	total := w.dir(path, 0)

	sort.SliceStable(w.folders, func(i, j int) bool {
		if w.folders[i].Bytes != w.folders[j].Bytes {
			return w.folders[i].Bytes > w.folders[j].Bytes
		}
		return w.folders[i].Path < w.folders[j].Path
	})
	if len(w.folders) > limit {
		w.folders = w.folders[:limit]
	}
	sort.Strings(w.bad)
	named := w.bad
	if len(named) > namedUnreadable {
		named = named[:namedUnreadable]
	}
	r := FolderReport{
		Path: path, Bytes: total, GiB: gibOf(total), Largest: w.folders,
		Partial: w.stopped != "", StoppedAt: w.stopped,
		Unreadable: len(w.bad), UnreadableFolders: named, Skipped: w.skipped,
	}
	if r.Largest == nil {
		r.Largest = []Folder{}
	}
	var said []string
	if r.Partial {
		said = append(said, fmt.Sprintf("PARTIAL: the walk stopped after %s at %s; sizes are what was measured so far", budget, r.StoppedAt))
	}
	said = append(said, fmt.Sprintf("%s holds %.1f GiB on its own filesystem", path, r.GiB))
	if r.Unreadable > 0 {
		said = append(said, fmt.Sprintf("%d folders could not be read (%s), so their size is missing", r.Unreadable, strings.Join(named, ", ")))
	}
	if len(r.Skipped) > 0 {
		said = append(said, "other filesystems not entered: "+strings.Join(r.Skipped, ", "))
	}
	r.Said = said
	return r, nil
}

func (w *walk) device(path string, info fs.FileInfo) uint64 {
	if w.m.DevOf != nil {
		return w.m.DevOf(path, info)
	}
	return devOf(path, info)
}

// dir is the allocated bytes under a folder; it records the folder when it is within the depth.
func (w *walk) dir(path string, depth int) int64 {
	if w.stopped != "" {
		return 0
	}
	if w.m.now().After(w.deadline) {
		w.stopped = path
		return 0
	}
	entries, err := w.m.readDir(w.m.path(path))
	if err != nil {
		w.bad = append(w.bad, path)
		return 0
	}
	var total int64
	for _, e := range entries {
		child := filepath.Join(path, e.Name())
		info, err := os.Lstat(w.m.path(child))
		if err != nil {
			continue // gone since it was listed
		}
		if info.IsDir() {
			if w.device(child, info) != w.dev {
				w.skipped = append(w.skipped, child)
				continue
			}
			total += w.allocated(info, false) + w.dir(child, depth+1)
			continue
		}
		total += w.allocated(info, true)
	}
	if depth >= 1 && depth <= w.maxDepth {
		w.folders = append(w.folders, Folder{Path: path, Depth: depth, Bytes: total, GiB: gibOf(total)})
	}
	return total
}

// allocated is the bytes an entry occupies on disk, as du counts them; a file with several links once.
func (w *walk) allocated(info fs.FileInfo, file bool) int64 {
	st, ok := info.Sys().(*syscall.Stat_t)
	if !ok {
		return info.Size()
	}
	if file && st.Nlink > 1 {
		k := inodeKey{uint64(st.Dev), uint64(st.Ino)}
		if w.seen[k] {
			return 0
		}
		w.seen[k] = true
	}
	return st.Blocks * 512
}

func gibOf(b int64) float64 { return math.Round(float64(b)/gib*100) / 100 }