buildstep: keep what one command of an ask writes into the caches for its later commands

Every command of a check ran over the shared Go build cache beneath a throwaway
tmpfs layer of its own, so the digests, the vet, the unit tests and the integration
compile of one ask each compiled every package again. With Isolation.CacheLayer a
command writes into a layer on disk of its own and Isolated moves it into the ask's
kept layer when it ends; later commands read the kept layer above the shared cache,
which stays unchanged (ADR 0541 §2, ADR 0448 §3). Empty CacheLayer keeps today's
throwaway layer. Ported from feat/try-speed S2 (issue 1921).

go/buildstep/isolate_test.go

@@ -2,7 +2,10 @@ package buildstep
 
 import (
 	"context"
+	"os"
+	"path/filepath"
 	"strings"
+	"sync"
 	"testing"
 
 	"git.novox.be/novox/mesh-sdk/go/forgekey"
@@ -112,4 +115,55 @@ func TestIsolatedRunsEachCommandThroughBubblewrap(t *testing.T) {
 	if hosts == "" {
 		t.Fatalf("no hosts file was bound: %v", ran)
 	}
+
+	// 1921/S2: with a cache layer, what one command wrote into the caches is kept for the ask's later commands, the
+	// shared cache stays unchanged, two commands at once write into layers of their own (overlayfs refuses an upper
+	// layer two mounts share), and a command's own layer is gone when it ends.
+	root := t.TempDir()
+	shared := filepath.Join(root, "gobuild")
+	if err := os.MkdirAll(shared, 0o700); err != nil {
+		t.Fatal(err)
+	}
+	layer := filepath.Join(root, "ws", "cachelayer")
+	var mu sync.Mutex
+	uppers := map[string]bool{}
+	gate := make(chan struct{})
+	var started sync.WaitGroup
+	started.Add(2)
+	writer := func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
+		up := overlayOf(args, shared)
+		mu.Lock()
+		uppers[up] = true
+		mu.Unlock()
+		started.Done()
+		<-gate // both commands are running before either ends
+		os.MkdirAll(filepath.Join(up, "ab"), 0o700)
+		os.WriteFile(filepath.Join(up, "ab", args[len(args)-1]+"-d"), []byte("compiled"), 0o600)
+		return forgekey.Ran{}
+	}
+	run := Isolated(writer, Isolation{Workspace: filepath.Join(root, "ws"), Caches: []string{shared}, CacheLayer: layer})
+	var wg sync.WaitGroup
+	for _, pkg := range []string{"one", "two"} {
+		wg.Add(1)
+		go func() { defer wg.Done(); run(context.Background(), nil, "go", "build", pkg) }()
+	}
+	started.Wait()
+	close(gate)
+	wg.Wait()
+	if len(uppers) != 2 {
+		t.Fatalf("two commands at once wrote into %d layers, want 2 of their own: %v", len(uppers), uppers)
+	}
+	for _, pkg := range []string{"one", "two"} {
+		if b, err := os.ReadFile(filepath.Join(keptLayer(layer, 0), "ab", pkg+"-d")); err != nil || string(b) != "compiled" {
+			t.Errorf("what the build of %s compiled is not kept for the ask: %q %v", pkg, b, err)
+		}
+	}
+	for up := range uppers {
+		if _, err := os.Stat(up); !os.IsNotExist(err) {
+			t.Errorf("the command's own layer %s outlived it: %v", up, err)
+		}
+	}
+	if entries, _ := os.ReadDir(shared); len(entries) != 0 {
+		t.Errorf("the shared cache was written: %v", entries)
+	}
 }