From 6147a204546734cb440f623b6621e74d6188e721 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 23 Oct 2026 08:28:28 +0200 Subject: [PATCH] Refuse a build step under a free-space floor and release workspaces, so a build node's disk is never filled (issue 1839) buildstep.Floor refuses a step, naming the node, what is free and the floor (20 GiB, build-agent's), while the work or cache filesystem is under it. buildstep.Workspaces holds a workspace while a step runs on it, removes it when its caller releases it, and sweeps one left unheld past ADR 0509's two-hour bound. --- go/buildstep/workspaces.go | 160 ++++++++++++++++++++++++++++++++ go/buildstep/workspaces_test.go | 113 ++++++++++++++++++++++ 2 files changed, 273 insertions(+) create mode 100644 go/buildstep/workspaces.go create mode 100644 go/buildstep/workspaces_test.go diff --git a/go/buildstep/workspaces.go b/go/buildstep/workspaces.go new file mode 100644 index 0000000..eefdee3 --- /dev/null +++ b/go/buildstep/workspaces.go @@ -0,0 +1,160 @@ +package buildstep + +// A build module's room to work and its workspaces (novox/hq issue 1839, ADR 0509): a step is refused while the +// filesystem it writes to has less free than the floor, and a workspace is removed when its caller releases it, or, +// unheld, once its last use is past a bound (a lost caller's), never while a running step holds it. + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "sync" + "syscall" + "time" +) + +// FreeSpaceFloor is the free space a build module needs on its work and cache filesystems to take a step: the floor +// build-agent keeps (mesh-controller internal/builder/floor.go, issue 1834). +const FreeSpaceFloor uint64 = 20 << 30 + +// UnheldBound is how long a workspace no step holds is kept after its last use, when its caller never released it +// (ADR 0509: a lost caller's workspace goes after two hours). +const UnheldBound = 2 * time.Hour + +// FreeBytes reads the bytes an unprivileged writer can still use on the filesystem holding path. +func FreeBytes(path string) (uint64, error) { + var s syscall.Statfs_t + if err := syscall.Statfs(path, &s); err != nil { + return 0, err + } + return s.Bavail * uint64(s.Bsize), nil +} + +// Floor refuses a step while a filesystem it writes to has less free than Bytes. A zero Bytes checks nothing. +type Floor struct { + // Node is named in the refusal. + Node string + Bytes uint64 + // Free reads a path's free bytes; FreeBytes when nil. A test answers for a disk it does not have. + Free func(path string) (uint64, error) +} + +// Check answers a *Refusal, "refused: has N GiB free under the floor of F GiB (on )", when any path's +// filesystem has less free than the floor, or its free space cannot be read; nil otherwise. A path not made yet is +// read at its nearest folder that is; an empty path is skipped. +func (f Floor) Check(paths ...string) error { + if f.Bytes == 0 { + return nil + } + free := f.Free + if free == nil { + free = FreeBytes + } + for _, p := range paths { + if p == "" { + continue + } + at := p + for { + if _, err := os.Stat(at); err == nil || filepath.Dir(at) == at { + break + } + at = filepath.Dir(at) + } + n, err := free(at) + if err != nil { + return &Refusal{Text: fmt.Sprintf("refused: the free space of %s on %s cannot be read: %v", p, f.Node, err)} + } + if n < f.Bytes { + return &Refusal{Text: fmt.Sprintf("refused: %s has %d GiB free under the floor of %d GiB (on %s)", f.Node, n>>30, f.Bytes>>30, p), + Want: fmt.Sprintf("%d GiB", f.Bytes>>30), Have: fmt.Sprintf("%d GiB", n>>30)} + } + } + return nil +} + +var workspaceID = regexp.MustCompile(`^[a-z0-9]+$`) + +// Workspaces are a build module's workspaces, each /, and which of them running steps hold. +type Workspaces struct { + Dir string + + mu sync.Mutex + held map[string]int +} + +// Hold marks the workspace held by a running step until the answered func is called; both mark it used now, so the +// sweep counts its bound from the step's end. An id that is not a workspace's holds nothing. +func (w *Workspaces) Hold(id string) func() { + if !workspaceID.MatchString(id) { + return func() {} + } + w.mu.Lock() + if w.held == nil { + w.held = map[string]int{} + } + w.held[id]++ + w.mu.Unlock() + w.touch(id) + var once sync.Once + return func() { + once.Do(func() { + w.touch(id) + w.mu.Lock() + if w.held[id]--; w.held[id] <= 0 { + delete(w.held, id) + } + w.mu.Unlock() + }) + } +} + +func (w *Workspaces) touch(id string) { + now := time.Now() + _ = os.Chtimes(filepath.Join(w.Dir, id), now, now) // a workspace not made yet has nothing to mark +} + +// Release removes the workspace at once: its caller's chain of steps has ended. One already gone is no error; one a +// running step holds is refused and kept; an id that is not a workspace's is refused. +func (w *Workspaces) Release(id string) error { + if !workspaceID.MatchString(id) { + return fmt.Errorf("%q is not a workspace id", id) + } + w.mu.Lock() + defer w.mu.Unlock() + if w.held[id] > 0 { + return fmt.Errorf("the workspace %s is held by a running step", id) + } + if err := os.RemoveAll(filepath.Join(w.Dir, id)); err != nil { + return fmt.Errorf("the workspace %s could not be removed: %w", id, err) + } + return nil +} + +// Sweep removes every workspace no step holds whose last use is older than bound, and answers the ids it removed, +// sorted. It is the safety net for a caller that never released its workspace. +func (w *Workspaces) Sweep(now time.Time, bound time.Duration) []string { + entries, err := os.ReadDir(w.Dir) + if err != nil { + return nil + } + removed := []string{} + w.mu.Lock() + defer w.mu.Unlock() + for _, e := range entries { + if !e.IsDir() || !workspaceID.MatchString(e.Name()) || w.held[e.Name()] > 0 { + continue + } + info, err := e.Info() + if err != nil || now.Sub(info.ModTime()) < bound { + continue + } + if os.RemoveAll(filepath.Join(w.Dir, e.Name())) == nil { + removed = append(removed, e.Name()) + } + } + sort.Strings(removed) + return removed +} diff --git a/go/buildstep/workspaces_test.go b/go/buildstep/workspaces_test.go new file mode 100644 index 0000000..8badfa2 --- /dev/null +++ b/go/buildstep/workspaces_test.go @@ -0,0 +1,113 @@ +package buildstep + +import ( + "errors" + "os" + "path/filepath" + "testing" + "time" +) + +// 1839/AC1: a filesystem under the floor refuses the step, naming the node, what is free and the floor. +func TestAFilesystemUnderTheFloorRefusesNamingNodeFreeAndFloor(t *testing.T) { + dir := t.TempDir() + f := Floor{Node: "ace", Bytes: 20 << 30, Free: func(string) (uint64, error) { return 4<<30 + 5, nil }} + err := f.Check(dir) + var ref *Refusal + if !errors.As(err, &ref) { + t.Fatalf("Check = %v, want a *Refusal", err) + } + if want := "refused: ace has 4 GiB free under the floor of 20 GiB (on " + dir + ")"; err.Error() != want { + t.Errorf("Check = %q, want %q", err, want) + } +} + +// 1839/AC1: at or above the floor, and with no floor set, nothing is refused. +func TestAtTheFloorOrWithNoFloorNothingIsRefused(t *testing.T) { + dir := t.TempDir() + if err := (Floor{Node: "ace", Bytes: 20 << 30, Free: func(string) (uint64, error) { return 20 << 30, nil }}).Check(dir); err != nil { + t.Errorf("at the floor: %v", err) + } + if err := (Floor{Node: "ace", Free: func(string) (uint64, error) { return 0, nil }}).Check(dir); err != nil { + t.Errorf("no floor: %v", err) + } +} + +// 1839/AC1: a path not made yet is read at its nearest folder that is; a free space that cannot be read is a refusal. +func TestAPathNotMadeIsReadAtItsParentAndAnUnreadableOneRefuses(t *testing.T) { + dir := t.TempDir() + var read string + f := Floor{Node: "ace", Bytes: 1, Free: func(p string) (uint64, error) { read = p; return 2, nil }} + if err := f.Check(filepath.Join(dir, "not", "made")); err != nil || read != dir { + t.Errorf("Check = %v, read %q, want nil and %q", err, read, dir) + } + f.Free = func(string) (uint64, error) { return 0, errors.New("statfs: boom") } + var ref *Refusal + if err := f.Check(dir); !errors.As(err, &ref) { + t.Errorf("Check = %v, want a refusal", err) + } +} + +func makeWorkspace(t *testing.T, w *Workspaces, id string, age time.Duration) string { + t.Helper() + p := filepath.Join(w.Dir, id) + if err := os.MkdirAll(filepath.Join(p, "tree"), 0o700); err != nil { + t.Fatal(err) + } + at := time.Now().Add(-age) + if err := os.Chtimes(p, at, at); err != nil { + t.Fatal(err) + } + return p +} + +func gone(p string) bool { _, err := os.Stat(p); return os.IsNotExist(err) } + +// 1839/AC2: a released workspace is gone at once; one already gone is no error; one a running step holds is refused +// and kept; an id that is not a workspace's is refused. +func TestReleaseRemovesAWorkspaceAtOnce(t *testing.T) { + w := &Workspaces{Dir: t.TempDir()} + p := makeWorkspace(t, w, "abc1", 0) + if err := w.Release("abc1"); err != nil || !gone(p) { + t.Fatalf("Release = %v, gone %v", err, gone(p)) + } + if err := w.Release("abc1"); err != nil { + t.Errorf("a second release = %v, want nil", err) + } + held := makeWorkspace(t, w, "abc2", 0) + done := w.Hold("abc2") + if err := w.Release("abc2"); err == nil || gone(held) { + t.Errorf("Release of a held workspace = %v, gone %v; want refused and kept", err, gone(held)) + } + done() + if err := w.Release("abc2"); err != nil || !gone(held) { + t.Errorf("Release after the hold ended = %v, gone %v", err, gone(held)) + } + if err := w.Release("../etc"); err == nil { + t.Errorf("Release of ../etc answered nil") + } +} + +// 1839/AC3: the sweep removes a workspace no step holds whose last use is past the bound, and keeps a held one and a +// recent one; a hold ending counts as a use. +func TestSweepRemovesUnheldWorkspacesPastTheBound(t *testing.T) { + w := &Workspaces{Dir: t.TempDir()} + old := makeWorkspace(t, w, "old1", 3*time.Hour) + heldOld := makeWorkspace(t, w, "held1", 3*time.Hour) + recent := makeWorkspace(t, w, "new1", time.Minute) + if err := os.WriteFile(filepath.Join(w.Dir, "not-a-workspace.txt"), nil, 0o600); err != nil { + t.Fatal(err) + } + done := w.Hold("held1") + removed := w.Sweep(time.Now(), 2*time.Hour) + if !gone(old) || gone(heldOld) || gone(recent) { + t.Errorf("after the sweep: old gone %v, held kept %v, recent kept %v", gone(old), !gone(heldOld), !gone(recent)) + } + if len(removed) != 1 || removed[0] != "old1" { + t.Errorf("Sweep answered %v, want [old1]", removed) + } + done() + if w.Sweep(time.Now(), 2*time.Hour); gone(heldOld) { + t.Errorf("a workspace whose hold just ended was swept: its end is a use") + } +}