mesh-sdk/go/buildstep/workspaces.go

package buildstep

// A build module's room to work and its workspaces (novox/hq issue 1839, ADR 0509): a step is refused while the
// filesystem it writes to has less free than the floor, and a workspace is removed when its caller releases it, or,
// unheld, once its last use is past a bound (a lost caller's), never while a running step holds it.

import (
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"sync"
	"syscall"
	"time"
)

// FreeSpaceFloor is the free space a build module needs on its work and cache filesystems to take a step: the floor
// build-agent keeps (mesh-controller internal/builder/floor.go, issue 1834).
const FreeSpaceFloor uint64 = 20 << 30

// UnheldBound is how long a workspace no step holds is kept after its last use, when its caller never released it
// (ADR 0509: a lost caller's workspace goes after two hours).
const UnheldBound = 2 * time.Hour

// FreeBytes reads the bytes an unprivileged writer can still use on the filesystem holding path.
func FreeBytes(path string) (uint64, error) {
	var s syscall.Statfs_t
	if err := syscall.Statfs(path, &s); err != nil {
		return 0, err
	}
	return s.Bavail * uint64(s.Bsize), nil
}

// Floor refuses a step while a filesystem it writes to has less free than Bytes. A zero Bytes checks nothing.
type Floor struct {
	// Node is named in the refusal.
	Node  string
	Bytes uint64
	// Free reads a path's free bytes; FreeBytes when nil. A test answers for a disk it does not have.
	Free func(path string) (uint64, error)
}

// Check answers a *Refusal, "refused: <node> has N GiB free under the floor of F GiB (on <path>)", when any path's
// filesystem has less free than the floor, or its free space cannot be read; nil otherwise. A path not made yet is
// read at its nearest folder that is; an empty path is skipped.
func (f Floor) Check(paths ...string) error {
	if f.Bytes == 0 {
		return nil
	}
	free := f.Free
	if free == nil {
		free = FreeBytes
	}
	for _, p := range paths {
		if p == "" {
			continue
		}
		at := p
		for {
			if _, err := os.Stat(at); err == nil || filepath.Dir(at) == at {
				break
			}
			at = filepath.Dir(at)
		}
		n, err := free(at)
		if err != nil {
			return &Refusal{Text: fmt.Sprintf("refused: the free space of %s on %s cannot be read: %v", p, f.Node, err)}
		}
		if n < f.Bytes {
			return &Refusal{Text: fmt.Sprintf("refused: %s has %d GiB free under the floor of %d GiB (on %s)", f.Node, n>>30, f.Bytes>>30, p),
				Want: fmt.Sprintf("%d GiB", f.Bytes>>30), Have: fmt.Sprintf("%d GiB", n>>30)}
		}
	}
	return nil
}

var workspaceID = regexp.MustCompile(`^[a-z0-9]+$`)

// Workspaces are a build module's workspaces, each <Dir>/<id>, and which of them running steps hold.
type Workspaces struct {
	Dir string

	mu   sync.Mutex
	held map[string]int
}

// Hold marks the workspace held by a running step until the answered func is called; both mark it used now, so the
// sweep counts its bound from the step's end. An id that is not a workspace's holds nothing.
func (w *Workspaces) Hold(id string) func() {
	if !workspaceID.MatchString(id) {
		return func() {}
	}
	w.mu.Lock()
	if w.held == nil {
		w.held = map[string]int{}
	}
	w.held[id]++
	w.mu.Unlock()
	w.touch(id)
	var once sync.Once
	return func() {
		once.Do(func() {
			w.touch(id)
			w.mu.Lock()
			if w.held[id]--; w.held[id] <= 0 {
				delete(w.held, id)
			}
			w.mu.Unlock()
		})
	}
}

func (w *Workspaces) touch(id string) {
	now := time.Now()
	_ = os.Chtimes(filepath.Join(w.Dir, id), now, now) // a workspace not made yet has nothing to mark
}

// cacheLayerSuffix names a workspace's cache layer beside it (Isolation.CacheLayer): never inside it, which every step
// of the workspace binds read-write.
const cacheLayerSuffix = ".cachelayer"

// CacheLayer is the directory of the workspace id's cache layer: beside the workspace, out of every step's reach, and
// removed with it by Release and Sweep.
func (w *Workspaces) CacheLayer(id string) string { return filepath.Join(w.Dir, id+cacheLayerSuffix) }

// Release removes the workspace at once: its caller's chain of steps has ended. One already gone is no error; one a
// running step holds is refused and kept; an id that is not a workspace's is refused.
func (w *Workspaces) Release(id string) error {
	if !workspaceID.MatchString(id) {
		return fmt.Errorf("%q is not a workspace id", id)
	}
	w.mu.Lock()
	defer w.mu.Unlock()
	if w.held[id] > 0 {
		return fmt.Errorf("the workspace %s is held by a running step", id)
	}
	if err := os.RemoveAll(filepath.Join(w.Dir, id)); err != nil {
		return fmt.Errorf("the workspace %s could not be removed: %w", id, err)
	}
	if err := os.RemoveAll(w.CacheLayer(id)); err != nil {
		return fmt.Errorf("the cache layer of the workspace %s could not be removed: %w", id, err)
	}
	return nil
}

// Sweep removes every workspace no step holds whose last use is older than bound, and answers the ids it removed,
// sorted. It is the safety net for a caller that never released its workspace.
func (w *Workspaces) Sweep(now time.Time, bound time.Duration) []string {
	entries, err := os.ReadDir(w.Dir)
	if err != nil {
		return nil
	}
	removed := []string{}
	w.mu.Lock()
	defer w.mu.Unlock()
	for _, e := range entries {
		if id, ok := strings.CutSuffix(e.Name(), cacheLayerSuffix); ok && workspaceID.MatchString(id) && w.held[id] == 0 {
			// a cache layer whose workspace is gone (its removal failed or was cut short) goes too
			if _, err := os.Stat(filepath.Join(w.Dir, id)); os.IsNotExist(err) {
				os.RemoveAll(filepath.Join(w.Dir, e.Name()))
			}
			continue
		}
		if !e.IsDir() || !workspaceID.MatchString(e.Name()) || w.held[e.Name()] > 0 {
			continue
		}
		info, err := e.Info()
		if err != nil || now.Sub(info.ModTime()) < bound {
			continue
		}
		if os.RemoveAll(filepath.Join(w.Dir, e.Name())) == nil {
			os.RemoveAll(w.CacheLayer(e.Name()))
			removed = append(removed, e.Name())
		}
	}
	sort.Strings(removed)
	return removed
}