mesh-build-go: give each ask a cache layer in its workspace, so what its digest and vet compile its tests find compiled

Pins mesh-sdk fix/check-speed-s2 (2a867018), whose Isolation.CacheLayer keeps what one
command of an ask writes into the caches for the ask's later commands, the shared cache
unchanged. A trunk build and the lint's vet still write the shared cache directly.
Ported from feat/try-speed S2 (issue 1921; ADR 0541 §2).

modules/mesh-build-go/cmd/mesh-build-go/fetch.go

@@ -296,6 +296,7 @@ func (m *Module) goEnv() []string {
 		modcache = m.GoModCache
 	}
 	return append([]string{
+		"CGO_ENABLED=0", // as a production build compiles (build.go), so a check tests what ships and the warm cache matches (ADR 0542 §1)
 		"GOTOOLCHAIN=local",
 		"GOFLAGS=-mod=mod",
 		"GOMODCACHE=" + modcache,

modules/mesh-build-go/cmd/mesh-build-go/integration.go

@@ -12,6 +12,7 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"git.novox.be/novox/mesh-sdk/go/buildstep"
@@ -62,30 +63,73 @@ func (m *Module) integrationHere(ctx context.Context, workspace, tree string, pa
 	if err != nil {
 		return nil, err
 	}
-	postgres, stopStore, err := m.startStore(ctx, run)
-	if err != nil {
-		return nil, fmt.Errorf("the integration level's store: %w", err)
-	}
-	defer stopStore()
-	nats, stopBus, err := m.startBus(ctx, filepath.Join(run, "nats"))
-	if err != nil {
-		return nil, fmt.Errorf("the integration level's bus: %w", err)
-	}
-	defer stopBus()
-	env := append([]string{"MESH_TEST_POSTGRES=" + postgres, "MESH_TEST_NATS=" + nats}, m.checkEnv(workspace, nil)...)
-	// Compiling counts as setup: the tests are built once, here, with the addresses a TestMain may read, and the run
-	// below finds them built. Whether they compile is for the run to say; its failure is not read here.
 	// Only the directories the selected packages lie in compile (ADR 0536 §3); no selection is every directory.
 	byDir, err := packagesByDir(tree, dirs, packages)
 	if err != nil {
 		return nil, err
 	}
+	// The store, the bus and the compile start together, and the directories compile side by side up to the bound
+	// (issue 1921, try speed S5): the setup is the longest of them, not their sum. The compile runs no test binary
+	// (-exec true), so it needs neither server and is given nothing a test reads; whether the tests compile is for the
+	// run to say, its failure is not read here, and the run below finds them built.
+	var (
+		wg                 sync.WaitGroup
+		postgres, nats     string
+		stopStore, stopBus func()
+		storeErr, busErr   error
+		crash              any
+		crashMu            sync.Mutex
+	)
+	guard := func() {
+		if p := recover(); p != nil {
+			crashMu.Lock()
+			crash = p
+			crashMu.Unlock()
+		}
+	}
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		defer guard()
+		postgres, stopStore, storeErr = m.startStore(ctx, run)
+	}()
+	go func() {
+		defer wg.Done()
+		defer guard()
+		nats, stopBus, busErr = m.startBus(ctx, filepath.Join(run, "nats"))
+	}()
+	slots := make(chan struct{}, m.parallel())
 	for _, d := range dirs {
 		if len(byDir[d]) == 0 {
 			continue
 		}
-		m.Run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), env...), "go", append([]string{"-C", filepath.Join(tree, d), "test", "-tags", strings.Join(integrationTags, ","), "-run", "^$", "-count=1"}, byDir[d]...)...)
+		wg.Add(1)
+		slots <- struct{}{}
+		go func() {
+			defer wg.Done()
+			defer func() { <-slots }()
+			defer guard()
+			m.Run(ctx, m.goEnvFor(filepath.Join(tree, d)), "go", append([]string{"-C", filepath.Join(tree, d), "test", "-tags", strings.Join(integrationTags, ","), "-exec", "true", "-count=1"}, byDir[d]...)...)
+		}()
 	}
+	wg.Wait()
+	// Whatever started is stopped however the level ends, a half-started store included.
+	if stopStore != nil {
+		defer stopStore()
+	}
+	if stopBus != nil {
+		defer stopBus()
+	}
+	if crash != nil {
+		panic(crash)
+	}
+	if storeErr != nil {
+		return nil, fmt.Errorf("the integration level's store: %w", storeErr)
+	}
+	if busErr != nil {
+		return nil, fmt.Errorf("the integration level's bus: %w", busErr)
+	}
+	env := append([]string{"MESH_TEST_POSTGRES=" + postgres, "MESH_TEST_NATS=" + nats}, m.checkEnv(workspace, nil)...)
 	setup := m.clock().Sub(began)
 
 	return m.runLevel(ctx, tree, levelOpts{

modules/mesh-build-go/cmd/mesh-build-go/integration_level_test.go

@@ -35,6 +35,9 @@ type integFake struct {
 	// command runs, with its line.
 	gomods string
 	onTest func(line string)
+	// onCompile is called, unlocked, as each compile of the setup runs, and onStart as the store starts (issue 1921, S5).
+	onCompile func(line string, env []string)
+	onStart   func()
 }
 
 func newIntegFake(t *testing.T, captured string) *integFake {
@@ -71,7 +74,10 @@ func (f *integFake) run(ctx context.Context, env []string, name string, args ...
 			return forgekey.Ran{Stdout: f.gomods}
 		}
 		return forgekey.Ran{Stdout: "go.mod\n"}
-	case name == "go" && strings.Contains(line, "-run ^$"):
+	case name == "go" && strings.Contains(line, "-exec true"):
+		if f.onCompile != nil {
+			f.onCompile(line, env)
+		}
 		f.tick("compile")
 	case name == "go":
 		if f.onTest != nil {
@@ -84,6 +90,9 @@ func (f *integFake) run(ctx context.Context, env []string, name string, args ...
 		}
 		return forgekey.Ran{Stdout: f.captured, Status: status}
 	case name == "pg_ctl" && strings.HasSuffix(line, " start"):
+		if f.onStart != nil {
+			f.onStart()
+		}
 		f.tick("start")
 		for i, a := range args {
 			if a == "-D" {
@@ -314,8 +323,8 @@ func TestAStoreThatDoesNotStartEndsTheStepFailedAndLeavesNothing(t *testing.T) {
 	if !strings.Contains(f.lines(), " stop") {
 		t.Errorf("the half-started store was not stopped:\n%s", f.lines())
 	}
-	if f.busDir != "" {
-		t.Errorf("the bus was started after the store failed")
+	if f.busDir != "" && !f.busDown {
+		t.Errorf("the bus, started beside the store, was not stopped after the store failed")
 	}
 	if entries, _ := os.ReadDir(filepath.Join(m.Work, "ws1")); len(entries) != 1 || entries[0].Name() != "tree" {
 		t.Errorf("the workspace holds %v, want only its tree", entries)

modules/mesh-build-go/cmd/mesh-build-go/isolate.go

@@ -29,23 +29,32 @@ func (m *Module) caches() []string {
 }
 
 // isolation is how a run step of the workspace is shut off: the workspace (its tree, its siblings, its run directories,
-// its packed artifacts) read-write, the caches beneath a throwaway layer unless write, and an environment of a PATH and
+// its packed artifacts) read-write, the caches beneath the ask's own layer unless write, and an environment of a PATH and
 // a locale alone (the step's HOME is its own, inside its /tmp).
 func (m *Module) isolation(workspace string, write bool) buildstep.Isolation {
 	path := os.Getenv("PATH")
 	if path == "" {
 		path = defaultPath
 	}
-	return buildstep.Isolation{
+	iso := buildstep.Isolation{
 		Workspace:   filepath.Join(m.Work, workspace),
 		Caches:      m.caches(),
 		WriteCaches: write,
 		Env:         append([]string{"PATH=" + path, "LC_ALL=C"}, m.threadEnv()...),
 	}
+	if !write {
+		// What one command of the ask compiles is kept for its later commands (the vet, the unit tests, the integration
+		// level's compile), in the workspace and gone with it (issue 1921, try speed S2).
+		iso.CacheLayer = filepath.Join(iso.Workspace, cacheLayerDir)
+	}
+	return iso
 }
 
+// cacheLayerDir is the directory in a workspace's root where an ask keeps what its commands wrote into the caches.
+const cacheLayerDir = "cachelayer"
+
 // stepRunner is the runner of a run step of the workspace: buildstep.Isolated over m.Run, so every command it runs is
-// shut off; write is true for a trunk build alone (the shared caches are written by trunk builds only). Inside is the
+// shut off; write is true for a trunk build and for a check's go vet, compile steps that run nothing of the change (ADR 0541 §2). Inside is the
 // integration level's own process, already running shut off (level.go): it runs its commands as they are.
 func (m *Module) stepRunner(workspace string, write bool) (forgekey.Runner, error) {
 	if m.Inside {

modules/mesh-build-go/cmd/mesh-build-go/isolate_test.go

@@ -20,6 +20,7 @@ import (
 type shutOff struct {
 	flags                    []string
 	binds, roBinds, overlays []string
+	uppers                   map[string]string // the writable layer each --overlay gives a cache: cache -> upper
 	env                      []string
 	command                  string
 	chdir                    string
@@ -27,7 +28,7 @@ type shutOff struct {
 
 // bwrapArity is how many words each bubblewrap option the SDK writes takes.
 var bwrapArity = map[string]int{"--unshare-all": 0, "--die-with-parent": 0, "--new-session": 0, "--clearenv": 0,
-	"--proc": 1, "--dev": 1, "--tmpfs": 1, "--dir": 1, "--chdir": 1, "--overlay-src": 1, "--tmp-overlay": 1,
+	"--proc": 1, "--dev": 1, "--tmpfs": 1, "--dir": 1, "--chdir": 1, "--overlay-src": 1, "--tmp-overlay": 1, "--overlay": 3,
 	"--ro-bind": 2, "--ro-bind-try": 2, "--bind": 2, "--symlink": 2, "--setenv": 2}
 
 // throughBubblewrap is the fake runner of a step's tests: a bubblewrap command is read back (recorded in seen) and its
@@ -56,6 +57,11 @@ func throughBubblewrap(t *testing.T, run forgekey.Runner, seen *[]shutOff) forge
 				s.roBinds = append(s.roBinds, args[i+1])
 			case "--overlay-src":
 				s.overlays = append(s.overlays, args[i+1])
+			case "--overlay":
+				if s.uppers == nil {
+					s.uppers = map[string]string{}
+				}
+				s.uppers[args[i+3]] = args[i+1]
 			case "--setenv":
 				s.env = append(s.env, args[i+1]+"="+args[i+2])
 			case "--chdir":
@@ -105,6 +111,9 @@ func checkShutOff(t *testing.T, m *Module, s shutOff, write bool) {
 			t.Errorf("a trunk build's %s does not write the shared cache %s (bound %v, layered %v)", s.command, c, bound, layered)
 		case !write && (bound || !layered):
 			t.Errorf("%s may write the shared cache %s (bound %v, layered %v): only a trunk build does", s.command, c, bound, layered)
+		case !write && !strings.HasPrefix(s.uppers[c], filepath.Join(m.Work, "ws1", "cachelayer")+"/"):
+			// 1921/S2: what one command of the ask compiles is kept for its later commands, in the workspace
+			t.Errorf("%s writes the cache %s into %q, not the ask's cache layer in its workspace", s.command, c, s.uppers[c])
 		}
 	}
 	if want := filepath.Join(m.Work, "ws1"); !slices.Contains(s.binds, want) {
@@ -130,8 +139,23 @@ func TestTheUnitLevelRunsEveryGoTestShutOff(t *testing.T) {
 	if ran == 0 || len(seen) != ran {
 		t.Fatalf("%d go commands ran, %d of them shut off: %v", ran, len(seen), *calls)
 	}
+	// A check compiles with cgo off as a build does (ADR 0542 §1), and the lint's go vet, a compile that runs nothing
+	// of the change, writes the shared caches; every other command reads them beneath its own layer (ADR 0541 §2).
+	for _, c := range *calls {
+		if strings.HasPrefix(c.line, "go ") && !slices.Contains(c.env, "CGO_ENABLED=0") {
+			t.Errorf("%s ran without CGO_ENABLED=0: %v", c.line, c.env)
+		}
+	}
+	vets := 0
 	for _, s := range seen {
-		checkShutOff(t, m, s, false)
+		vet := strings.Contains(s.command, " vet ")
+		if vet {
+			vets++
+		}
+		checkShutOff(t, m, s, vet)
+	}
+	if vets == 0 {
+		t.Error("no go vet ran shut off")
 	}
 }
 

modules/mesh-build-go/cmd/mesh-build-go/lint.go

@@ -13,6 +13,7 @@ import (
 	"sort"
 	"strings"
 	"sync"
+	"time"
 
 	"git.novox.be/novox/mesh-sdk/go/buildstep"
 	"git.novox.be/novox/mesh-sdk/go/forgekey"
@@ -144,6 +145,18 @@ func (m *Module) lintAll(ctx context.Context, run forgekey.Runner, tree string,
 	var wg sync.WaitGroup
 	var crash any // a panic of a lint goes on in the caller, as a panic of a test run does
 	var crashMu sync.Mutex
+	// Each directory's end is said as a phase, so a long lint is readable while it runs (issue 1921): which directories
+	// are done and how long the last one took.
+	var doneMu sync.Mutex
+	done := 0
+	said := func(d string, began time.Time) {
+		doneMu.Lock()
+		defer doneMu.Unlock()
+		done++
+		if done < len(dirs) {
+			buildstep.Phase(ctx, fmt.Sprintf("unit: lint of %d directories, %d done (%s in %s)", len(dirs), done, d, m.clock().Sub(began).Round(time.Second)))
+		}
+	}
 	for i, d := range dirs {
 		wg.Add(1)
 		slots <- struct{}{}
@@ -157,7 +170,9 @@ func (m *Module) lintAll(ctx context.Context, run forgekey.Runner, tree string,
 					crashMu.Unlock()
 				}
 			}()
+			began := m.clock()
 			answers[i].failed, answers[i].err = m.lint(ctx, run, tree, d, env(d))
+			said(d, began)
 		}()
 	}
 	wg.Wait()

modules/mesh-build-go/cmd/mesh-build-go/preview_test.go

@@ -290,8 +290,8 @@ func TestTheIntegrationLevelIsGivenTheBesideDirectoryOnlyWhenSiblingsWerePlaced(
 		}
 		tested := 0
 		for _, c := range f.calls {
-			if !strings.HasPrefix(c.line, "go ") || !strings.Contains(c.line, "-tags integration") {
-				continue
+			if !strings.HasPrefix(c.line, "go ") || !strings.Contains(c.line, "-tags integration") || strings.Contains(c.line, "-exec true") {
+				continue // the setup's compile runs no test binary, so it is given nothing a test reads (issue 1921, S5)
 			}
 			tested++
 			got := ""

modules/mesh-build-go/cmd/mesh-build-go/selected_dirs_test.go

@@ -72,7 +72,7 @@ func TestOnlyTheSelectedDirectoryCompilesAndRuns(t *testing.T) {
 		if !strings.HasPrefix(l, "go -C "+filepath.Join(tree, "b")+" test ") {
 			t.Errorf("go test ran outside the selected directory: %s", l)
 		}
-		if strings.Contains(l, "-run ^$") {
+		if strings.Contains(l, "-exec true") {
 			compiled++
 		} else {
 			ran++
@@ -91,7 +91,7 @@ func TestWithoutASelectionEveryDirectoryCompilesAndRuns(t *testing.T) {
 	compiled, ran := 0, 0
 	for _, l := range strings.Split(f.lines(), "\n") {
 		if strings.HasPrefix(l, "go ") && strings.Contains(l, " test ") {
-			if strings.Contains(l, "-run ^$") {
+			if strings.Contains(l, "-exec true") {
 				compiled++
 			} else {
 				ran++
@@ -146,27 +146,85 @@ func TestSelectedDirectoriesRunAtOnceAndAnswerInDirectoryOrder(t *testing.T) {
 	}
 }
 
-// The bound holds: with one directory at a time, no two go test commands run at once.
+// The bound holds for the setup's compiles and for the tests: no more of them run at once than the bound, and with a
+// bound of 3 the three directories compile at once, beside the store's start, given nothing a test reads, since the
+// compile runs no test binary (issue 1921, try speed S5).
 func TestTheDirectoriesAtOnceNeverPassTheBound(t *testing.T) {
-	f := newIntegFake(t, "slow")
-	var mu sync.Mutex
-	running, most := 0, 0
-	f.onTest = func(string) {
-		mu.Lock()
-		running++
-		if running > most {
-			most = running
-		}
-		mu.Unlock()
-		time.Sleep(20 * time.Millisecond)
-		mu.Lock()
-		running--
-		mu.Unlock()
-	}
-	m, states := threeModules(t, f)
-	m.Parallel = 1
-	runIntegrationOf(t, m, states)
-	if most != 1 {
-		t.Errorf("%d go test commands ran at once with a bound of 1", most)
+	for _, tc := range []struct {
+		name    string
+		compile bool // the setup's compiles are counted, not the tests
+		bound   int
+		want    int // how many must run at once (at most the bound always)
+	}{
+		{"tests one at a time", false, 1, 1},
+		{"compiles one at a time", true, 1, 1},
+		{"compiles at once beside the store", true, 3, 3},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			f := newIntegFake(t, "slow")
+			var mu sync.Mutex
+			running, most := 0, 0
+			reached := make(chan struct{})
+			storeStarting, beside := false, false
+			count := func() {
+				mu.Lock()
+				running++
+				most = max(most, running)
+				if running == tc.want && tc.want > 1 {
+					close(reached)
+				}
+				mu.Unlock()
+				if tc.want > 1 {
+					select { // held until all of them run, so "at once" does not hang on timing
+					case <-reached:
+					case <-time.After(2 * time.Second):
+					}
+				} else {
+					time.Sleep(20 * time.Millisecond)
+				}
+				mu.Lock()
+				running--
+				mu.Unlock()
+			}
+			if tc.compile {
+				f.onCompile = func(_ string, env []string) {
+					for _, e := range env {
+						if strings.HasPrefix(e, "MESH_TEST_") {
+							t.Errorf("the compile was given %s: it runs no test binary", e)
+						}
+					}
+					mu.Lock()
+					beside = beside || storeStarting
+					mu.Unlock()
+					count()
+				}
+				if tc.want > 1 {
+					f.onStart = func() { // the store starts until the compiles have all run at once, or 2 s pass
+						mu.Lock()
+						storeStarting = true
+						mu.Unlock()
+						select {
+						case <-reached:
+						case <-time.After(2 * time.Second):
+						}
+					}
+				}
+			} else {
+				f.onTest = func(string) { count() }
+			}
+			m, states := threeModules(t, f)
+			m.Parallel = tc.bound
+			if _, st := runIntegrationOf(t, m, states); st.State != delivery.StepPassed {
+				t.Fatalf("the step ended %s: %s", st.State, st.Reason)
+			}
+			mu.Lock()
+			defer mu.Unlock()
+			if most > tc.bound || most < tc.want {
+				t.Errorf("%d ran at once with a bound of %d, want %d", most, tc.bound, tc.want)
+			}
+			if tc.compile && tc.want > 1 && !beside {
+				t.Errorf("no compile ran while the store was starting:\n%s", f.lines())
+			}
+		})
 	}
 }

modules/mesh-build-go/cmd/mesh-build-go/test.go

@@ -251,8 +251,15 @@ func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestR
 	if o.level == LevelUnit {
 		// gofmt and go vet of each directory the level runs come before its tests (ADR 0529 §1; issue 1887 D7-go): a
 		// directory that fails them fails the level at once, and their time is the result's Lint.
+		// go vet compiles and runs nothing of the change, with cgo off (goEnv), so it writes the shared cache and warms it
+		// for every later check, a pull request's head included (ADR 0541 §2, ADR 0542 §1); the tests run beneath their
+		// own throwaway layer.
+		compile, err := m.stepRunner(o.workspace, true)
+		if err != nil {
+			return nil, err
+		}
 		linted := m.clock()
-		failed, err := m.lintAll(ctx, run, tree, todo, func(d string) []string {
+		failed, err := m.lintAll(ctx, compile, tree, todo, func(d string) []string {
 			return append(m.goEnvFor(filepath.Join(tree, d)), o.env...)
 		})
 		if err != nil {

modules/mesh-build-go/go.mod

@@ -3,7 +3,7 @@ module meshbuildgo
 go 1.26.0
 
 require (
-	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028190821-9e5666b4d6f5
+	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028235126-2a8670189188
 	github.com/distribution/distribution/v3 v3.1.2
 	github.com/nats-io/nats-server/v2 v2.11.17
 	github.com/nats-io/nats.go v1.54.0

modules/mesh-build-go/go.sum

@@ -1,5 +1,7 @@
 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028190821-9e5666b4d6f5 h1:L7EJbCPZfJRCYHyco5Ck3ui+pWc//sp+h5nnPqSarng=
 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028190821-9e5666b4d6f5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028235126-2a8670189188 h1:0ZJZmp4wqiJlBlVHAdGonVIRaLf0Z79Ql4DFR2k8E8c=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028235126-2a8670189188/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
 github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI=
 github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=