buildstep: keep a check ask's cache layer out of every step's reach, move through an os.Root, and layer the build cache alone

Review of 62a50eb: the layer lay in the workspace, which every step binds read-write,
so a change's command could swap a kept directory for a symlink and the move, run
outside the sandbox, would write into the shared cache (ADR 0541 §2). The layer now
lies beside the workspace (Workspaces.CacheLayer, removed by Release and Sweep),
Command refuses a layer within reach of the workspace, a writable directory or a
cache, and every move goes through an os.Root on the layer and keeps no symlink. Only
LayerCaches (the build cache) are layered; a module cache keeps its throwaway layer
(issue 1921).

go/buildstep/workspaces.go

@@ -10,6 +10,7 @@ import (
 	"path/filepath"
 	"regexp"
 	"sort"
+	"strings"
 	"sync"
 	"syscall"
 	"time"
@@ -116,6 +117,14 @@ func (w *Workspaces) touch(id string) {
 	_ = os.Chtimes(filepath.Join(w.Dir, id), now, now) // a workspace not made yet has nothing to mark
 }
 
+// cacheLayerSuffix names a workspace's cache layer beside it (Isolation.CacheLayer): never inside it, which every step
+// of the workspace binds read-write.
+const cacheLayerSuffix = ".cachelayer"
+
+// CacheLayer is the directory of the workspace id's cache layer: beside the workspace, out of every step's reach, and
+// removed with it by Release and Sweep.
+func (w *Workspaces) CacheLayer(id string) string { return filepath.Join(w.Dir, id+cacheLayerSuffix) }
+
 // Release removes the workspace at once: its caller's chain of steps has ended. One already gone is no error; one a
 // running step holds is refused and kept; an id that is not a workspace's is refused.
 func (w *Workspaces) Release(id string) error {
@@ -130,6 +139,9 @@ func (w *Workspaces) Release(id string) error {
 	if err := os.RemoveAll(filepath.Join(w.Dir, id)); err != nil {
 		return fmt.Errorf("the workspace %s could not be removed: %w", id, err)
 	}
+	if err := os.RemoveAll(w.CacheLayer(id)); err != nil {
+		return fmt.Errorf("the cache layer of the workspace %s could not be removed: %w", id, err)
+	}
 	return nil
 }
 
@@ -144,6 +156,13 @@ func (w *Workspaces) Sweep(now time.Time, bound time.Duration) []string {
 	w.mu.Lock()
 	defer w.mu.Unlock()
 	for _, e := range entries {
+		if id, ok := strings.CutSuffix(e.Name(), cacheLayerSuffix); ok && workspaceID.MatchString(id) && w.held[id] == 0 {
+			// a cache layer whose workspace is gone (its removal failed or was cut short) goes too
+			if _, err := os.Stat(filepath.Join(w.Dir, id)); os.IsNotExist(err) {
+				os.RemoveAll(filepath.Join(w.Dir, e.Name()))
+			}
+			continue
+		}
 		if !e.IsDir() || !workspaceID.MatchString(e.Name()) || w.held[e.Name()] > 0 {
 			continue
 		}
@@ -152,6 +171,7 @@ func (w *Workspaces) Sweep(now time.Time, bound time.Duration) []string {
 			continue
 		}
 		if os.RemoveAll(filepath.Join(w.Dir, e.Name())) == nil {
+			os.RemoveAll(w.CacheLayer(e.Name()))
 			removed = append(removed, e.Name())
 		}
 	}