Refuse a build step under a free-space floor and release workspaces, so a build node's disk is never filled (issue 1839)

buildstep.Floor refuses a step, naming the node, what is free and the floor
(20 GiB, build-agent's), while the work or cache filesystem is under it.
buildstep.Workspaces holds a workspace while a step runs on it, removes it
when its caller releases it, and sweeps one left unheld past ADR 0509's
two-hour bound.

go/buildstep/workspaces.go

@@ -0,0 +1,160 @@
+package buildstep
+
+// A build module's room to work and its workspaces (novox/hq issue 1839, ADR 0509): a step is refused while the
+// filesystem it writes to has less free than the floor, and a workspace is removed when its caller releases it, or,
+// unheld, once its last use is past a bound (a lost caller's), never while a running step holds it.
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"sync"
+	"syscall"
+	"time"
+)
+
+// FreeSpaceFloor is the free space a build module needs on its work and cache filesystems to take a step: the floor
+// build-agent keeps (mesh-controller internal/builder/floor.go, issue 1834).
+const FreeSpaceFloor uint64 = 20 << 30
+
+// UnheldBound is how long a workspace no step holds is kept after its last use, when its caller never released it
+// (ADR 0509: a lost caller's workspace goes after two hours).
+const UnheldBound = 2 * time.Hour
+
+// FreeBytes reads the bytes an unprivileged writer can still use on the filesystem holding path.
+func FreeBytes(path string) (uint64, error) {
+	var s syscall.Statfs_t
+	if err := syscall.Statfs(path, &s); err != nil {
+		return 0, err
+	}
+	return s.Bavail * uint64(s.Bsize), nil
+}
+
+// Floor refuses a step while a filesystem it writes to has less free than Bytes. A zero Bytes checks nothing.
+type Floor struct {
+	// Node is named in the refusal.
+	Node  string
+	Bytes uint64
+	// Free reads a path's free bytes; FreeBytes when nil. A test answers for a disk it does not have.
+	Free func(path string) (uint64, error)
+}
+
+// Check answers a *Refusal, "refused: <node> has N GiB free under the floor of F GiB (on <path>)", when any path's
+// filesystem has less free than the floor, or its free space cannot be read; nil otherwise. A path not made yet is
+// read at its nearest folder that is; an empty path is skipped.
+func (f Floor) Check(paths ...string) error {
+	if f.Bytes == 0 {
+		return nil
+	}
+	free := f.Free
+	if free == nil {
+		free = FreeBytes
+	}
+	for _, p := range paths {
+		if p == "" {
+			continue
+		}
+		at := p
+		for {
+			if _, err := os.Stat(at); err == nil || filepath.Dir(at) == at {
+				break
+			}
+			at = filepath.Dir(at)
+		}
+		n, err := free(at)
+		if err != nil {
+			return &Refusal{Text: fmt.Sprintf("refused: the free space of %s on %s cannot be read: %v", p, f.Node, err)}
+		}
+		if n < f.Bytes {
+			return &Refusal{Text: fmt.Sprintf("refused: %s has %d GiB free under the floor of %d GiB (on %s)", f.Node, n>>30, f.Bytes>>30, p),
+				Want: fmt.Sprintf("%d GiB", f.Bytes>>30), Have: fmt.Sprintf("%d GiB", n>>30)}
+		}
+	}
+	return nil
+}
+
+var workspaceID = regexp.MustCompile(`^[a-z0-9]+$`)
+
+// Workspaces are a build module's workspaces, each <Dir>/<id>, and which of them running steps hold.
+type Workspaces struct {
+	Dir string
+
+	mu   sync.Mutex
+	held map[string]int
+}
+
+// Hold marks the workspace held by a running step until the answered func is called; both mark it used now, so the
+// sweep counts its bound from the step's end. An id that is not a workspace's holds nothing.
+func (w *Workspaces) Hold(id string) func() {
+	if !workspaceID.MatchString(id) {
+		return func() {}
+	}
+	w.mu.Lock()
+	if w.held == nil {
+		w.held = map[string]int{}
+	}
+	w.held[id]++
+	w.mu.Unlock()
+	w.touch(id)
+	var once sync.Once
+	return func() {
+		once.Do(func() {
+			w.touch(id)
+			w.mu.Lock()
+			if w.held[id]--; w.held[id] <= 0 {
+				delete(w.held, id)
+			}
+			w.mu.Unlock()
+		})
+	}
+}
+
+func (w *Workspaces) touch(id string) {
+	now := time.Now()
+	_ = os.Chtimes(filepath.Join(w.Dir, id), now, now) // a workspace not made yet has nothing to mark
+}
+
+// Release removes the workspace at once: its caller's chain of steps has ended. One already gone is no error; one a
+// running step holds is refused and kept; an id that is not a workspace's is refused.
+func (w *Workspaces) Release(id string) error {
+	if !workspaceID.MatchString(id) {
+		return fmt.Errorf("%q is not a workspace id", id)
+	}
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	if w.held[id] > 0 {
+		return fmt.Errorf("the workspace %s is held by a running step", id)
+	}
+	if err := os.RemoveAll(filepath.Join(w.Dir, id)); err != nil {
+		return fmt.Errorf("the workspace %s could not be removed: %w", id, err)
+	}
+	return nil
+}
+
+// Sweep removes every workspace no step holds whose last use is older than bound, and answers the ids it removed,
+// sorted. It is the safety net for a caller that never released its workspace.
+func (w *Workspaces) Sweep(now time.Time, bound time.Duration) []string {
+	entries, err := os.ReadDir(w.Dir)
+	if err != nil {
+		return nil
+	}
+	removed := []string{}
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	for _, e := range entries {
+		if !e.IsDir() || !workspaceID.MatchString(e.Name()) || w.held[e.Name()] > 0 {
+			continue
+		}
+		info, err := e.Info()
+		if err != nil || now.Sub(info.ModTime()) < bound {
+			continue
+		}
+		if os.RemoveAll(filepath.Join(w.Dir, e.Name())) == nil {
+			removed = append(removed, e.Name())
+		}
+	}
+	sort.Strings(removed)
+	return removed
+}

go/buildstep/workspaces_test.go

@@ -0,0 +1,113 @@
+package buildstep
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+// 1839/AC1: a filesystem under the floor refuses the step, naming the node, what is free and the floor.
+func TestAFilesystemUnderTheFloorRefusesNamingNodeFreeAndFloor(t *testing.T) {
+	dir := t.TempDir()
+	f := Floor{Node: "ace", Bytes: 20 << 30, Free: func(string) (uint64, error) { return 4<<30 + 5, nil }}
+	err := f.Check(dir)
+	var ref *Refusal
+	if !errors.As(err, &ref) {
+		t.Fatalf("Check = %v, want a *Refusal", err)
+	}
+	if want := "refused: ace has 4 GiB free under the floor of 20 GiB (on " + dir + ")"; err.Error() != want {
+		t.Errorf("Check = %q, want %q", err, want)
+	}
+}
+
+// 1839/AC1: at or above the floor, and with no floor set, nothing is refused.
+func TestAtTheFloorOrWithNoFloorNothingIsRefused(t *testing.T) {
+	dir := t.TempDir()
+	if err := (Floor{Node: "ace", Bytes: 20 << 30, Free: func(string) (uint64, error) { return 20 << 30, nil }}).Check(dir); err != nil {
+		t.Errorf("at the floor: %v", err)
+	}
+	if err := (Floor{Node: "ace", Free: func(string) (uint64, error) { return 0, nil }}).Check(dir); err != nil {
+		t.Errorf("no floor: %v", err)
+	}
+}
+
+// 1839/AC1: a path not made yet is read at its nearest folder that is; a free space that cannot be read is a refusal.
+func TestAPathNotMadeIsReadAtItsParentAndAnUnreadableOneRefuses(t *testing.T) {
+	dir := t.TempDir()
+	var read string
+	f := Floor{Node: "ace", Bytes: 1, Free: func(p string) (uint64, error) { read = p; return 2, nil }}
+	if err := f.Check(filepath.Join(dir, "not", "made")); err != nil || read != dir {
+		t.Errorf("Check = %v, read %q, want nil and %q", err, read, dir)
+	}
+	f.Free = func(string) (uint64, error) { return 0, errors.New("statfs: boom") }
+	var ref *Refusal
+	if err := f.Check(dir); !errors.As(err, &ref) {
+		t.Errorf("Check = %v, want a refusal", err)
+	}
+}
+
+func makeWorkspace(t *testing.T, w *Workspaces, id string, age time.Duration) string {
+	t.Helper()
+	p := filepath.Join(w.Dir, id)
+	if err := os.MkdirAll(filepath.Join(p, "tree"), 0o700); err != nil {
+		t.Fatal(err)
+	}
+	at := time.Now().Add(-age)
+	if err := os.Chtimes(p, at, at); err != nil {
+		t.Fatal(err)
+	}
+	return p
+}
+
+func gone(p string) bool { _, err := os.Stat(p); return os.IsNotExist(err) }
+
+// 1839/AC2: a released workspace is gone at once; one already gone is no error; one a running step holds is refused
+// and kept; an id that is not a workspace's is refused.
+func TestReleaseRemovesAWorkspaceAtOnce(t *testing.T) {
+	w := &Workspaces{Dir: t.TempDir()}
+	p := makeWorkspace(t, w, "abc1", 0)
+	if err := w.Release("abc1"); err != nil || !gone(p) {
+		t.Fatalf("Release = %v, gone %v", err, gone(p))
+	}
+	if err := w.Release("abc1"); err != nil {
+		t.Errorf("a second release = %v, want nil", err)
+	}
+	held := makeWorkspace(t, w, "abc2", 0)
+	done := w.Hold("abc2")
+	if err := w.Release("abc2"); err == nil || gone(held) {
+		t.Errorf("Release of a held workspace = %v, gone %v; want refused and kept", err, gone(held))
+	}
+	done()
+	if err := w.Release("abc2"); err != nil || !gone(held) {
+		t.Errorf("Release after the hold ended = %v, gone %v", err, gone(held))
+	}
+	if err := w.Release("../etc"); err == nil {
+		t.Errorf("Release of ../etc answered nil")
+	}
+}
+
+// 1839/AC3: the sweep removes a workspace no step holds whose last use is past the bound, and keeps a held one and a
+// recent one; a hold ending counts as a use.
+func TestSweepRemovesUnheldWorkspacesPastTheBound(t *testing.T) {
+	w := &Workspaces{Dir: t.TempDir()}
+	old := makeWorkspace(t, w, "old1", 3*time.Hour)
+	heldOld := makeWorkspace(t, w, "held1", 3*time.Hour)
+	recent := makeWorkspace(t, w, "new1", time.Minute)
+	if err := os.WriteFile(filepath.Join(w.Dir, "not-a-workspace.txt"), nil, 0o600); err != nil {
+		t.Fatal(err)
+	}
+	done := w.Hold("held1")
+	removed := w.Sweep(time.Now(), 2*time.Hour)
+	if !gone(old) || gone(heldOld) || gone(recent) {
+		t.Errorf("after the sweep: old gone %v, held kept %v, recent kept %v", gone(old), !gone(heldOld), !gone(recent))
+	}
+	if len(removed) != 1 || removed[0] != "old1" {
+		t.Errorf("Sweep answered %v, want [old1]", removed)
+	}
+	done()
+	if w.Sweep(time.Now(), 2*time.Hour); gone(heldOld) {
+		t.Errorf("a workspace whose hold just ended was swept: its end is a use")
+	}
+}