mesh-build-go: start the integration level's store and bus beside its compile, and compile the go.mod directories side by side without running a test binary (issue 1921, try speed S5)

modules/mesh-build-go/cmd/mesh-build-go/integration.go

@@ -12,6 +12,7 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"git.novox.be/novox/mesh-sdk/go/buildstep"
@@ -62,30 +63,67 @@ func (m *Module) integrationHere(ctx context.Context, workspace, tree string, pa
 	if err != nil {
 		return nil, err
 	}
-	postgres, stopStore, err := m.startStore(ctx, run)
-	if err != nil {
-		return nil, fmt.Errorf("the integration level's store: %w", err)
-	}
-	defer stopStore()
-	nats, stopBus, err := m.startBus(ctx, filepath.Join(run, "nats"))
-	if err != nil {
-		return nil, fmt.Errorf("the integration level's bus: %w", err)
-	}
-	defer stopBus()
-	env := append([]string{"MESH_TEST_POSTGRES=" + postgres, "MESH_TEST_NATS=" + nats}, m.besideEnv(workspace)...)
-	// Compiling counts as setup: the tests are built once, here, with the addresses a TestMain may read, and the run
-	// below finds them built. Whether they compile is for the run to say; its failure is not read here.
 	// Only the directories the selected packages lie in compile (ADR 0536 §3); no selection is every directory.
 	byDir, err := packagesByDir(tree, dirs, packages)
 	if err != nil {
 		return nil, err
 	}
+	// The store, the bus and the compile start together (issue 1921, try speed S5); the setup is the longest of them.
+	// The compile runs no test binary (-exec true), so it needs neither server: whether the tests compile is for the run
+	// to say, and its failure is not read here; the run below finds them built.
+	var (
+		wg                 sync.WaitGroup
+		postgres, nats     string
+		stopStore, stopBus func()
+		storeErr, busErr   error
+		crash              any
+		crashMu            sync.Mutex
+	)
+	guard := func() {
+		if p := recover(); p != nil {
+			crashMu.Lock()
+			crash = p
+			crashMu.Unlock()
+		}
+	}
+	wg.Add(2)
+	go func() { defer wg.Done(); defer guard(); postgres, stopStore, storeErr = m.startStore(ctx, run) }()
+	go func() {
+		defer wg.Done()
+		defer guard()
+		nats, stopBus, busErr = m.startBus(ctx, filepath.Join(run, "nats"))
+	}()
+	slots := make(chan struct{}, m.parallel())
 	for _, d := range dirs {
 		if len(byDir[d]) == 0 {
 			continue
 		}
-		m.Run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), env...), "go", append([]string{"-C", filepath.Join(tree, d), "test", "-tags", strings.Join(integrationTags, ","), "-run", "^$", "-count=1"}, byDir[d]...)...)
+		wg.Add(1)
+		slots <- struct{}{}
+		go func() {
+			defer wg.Done()
+			defer func() { <-slots }()
+			defer guard()
+			m.Run(ctx, m.goEnvFor(filepath.Join(tree, d)), "go", append([]string{"-C", filepath.Join(tree, d), "test", "-tags", strings.Join(integrationTags, ","), "-exec", "true", "-count=1"}, byDir[d]...)...)
+		}()
 	}
+	wg.Wait()
+	if stopStore != nil {
+		defer stopStore()
+	}
+	if stopBus != nil {
+		defer stopBus()
+	}
+	if crash != nil {
+		panic(crash)
+	}
+	if storeErr != nil {
+		return nil, fmt.Errorf("the integration level's store: %w", storeErr)
+	}
+	if busErr != nil {
+		return nil, fmt.Errorf("the integration level's bus: %w", busErr)
+	}
+	env := append([]string{"MESH_TEST_POSTGRES=" + postgres, "MESH_TEST_NATS=" + nats}, m.besideEnv(workspace)...)
 	setup := m.clock().Sub(began)
 
 	return m.runLevel(ctx, tree, levelOpts{

modules/mesh-build-go/cmd/mesh-build-go/integration_level_test.go

@@ -35,6 +35,8 @@ type integFake struct {
 	// command runs, with its line.
 	gomods string
 	onTest func(line string)
+	// onCompile is called, unlocked, as each compile of the setup runs (issue 1921, try speed S5).
+	onCompile func(line string)
 }
 
 func newIntegFake(t *testing.T, captured string) *integFake {
@@ -71,7 +73,10 @@ func (f *integFake) run(ctx context.Context, env []string, name string, args ...
 			return forgekey.Ran{Stdout: f.gomods}
 		}
 		return forgekey.Ran{Stdout: "go.mod\n"}
-	case name == "go" && strings.Contains(line, "-run ^$"):
+	case name == "go" && strings.Contains(line, "-exec true"):
+		if f.onCompile != nil {
+			f.onCompile(line)
+		}
 		f.tick("compile")
 	case name == "go":
 		if f.onTest != nil {
@@ -314,8 +319,8 @@ func TestAStoreThatDoesNotStartEndsTheStepFailedAndLeavesNothing(t *testing.T) {
 	if !strings.Contains(f.lines(), " stop") {
 		t.Errorf("the half-started store was not stopped:\n%s", f.lines())
 	}
-	if f.busDir != "" {
-		t.Errorf("the bus was started after the store failed")
+	if f.busDir != "" && !f.busDown {
+		t.Errorf("the bus, started beside the store, was not stopped after the store failed")
 	}
 	if entries, _ := os.ReadDir(filepath.Join(m.Work, "ws1")); len(entries) != 1 || entries[0].Name() != "tree" {
 		t.Errorf("the workspace holds %v, want only its tree", entries)
@@ -377,3 +382,68 @@ func TestTheSocketPathStaysUnderTheLimitForALongRunDirectory(t *testing.T) {
 		t.Errorf("the short socket directory %s outlived the store (%v)", sock, err)
 	}
 }
+
+// 1921/S5: the setup compiles the go.mod directories side by side, up to the bound, beside the store's and the bus's
+// start, and compiles without running a test binary, so it needs neither; the setup stays one number.
+func TestTheSetupCompilesTheDirectoriesAtOnceBesideTheStoreAndTheBus(t *testing.T) {
+	f := newIntegFake(t, "slow")
+	f.gomods = "a/go.mod\nb/go.mod\nc/go.mod\nd/go.mod\ne/go.mod\n"
+	var mu sync.Mutex
+	running, most := 0, 0
+	storeStarting, storeUp, overlap := false, false, false
+	f.onCompile = func(line string) {
+		mu.Lock()
+		running++
+		most = max(most, running)
+		mu.Unlock()
+		time.Sleep(40 * time.Millisecond)
+		mu.Lock()
+		running--
+		mu.Unlock()
+	}
+	m, states := integModule(t, f)
+	m.Parallel = 3
+	tree := filepath.Join(m.Work, "ws1", "tree")
+	for _, d := range []string{"a", "b", "c", "d", "e"} {
+		os.MkdirAll(filepath.Join(tree, d), 0o755)
+		os.WriteFile(filepath.Join(tree, d, "go.mod"), []byte("module x/"+d+"\n"), 0o644)
+	}
+	inner := m.Run
+	m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
+		line := strings.Join(args, " ")
+		if name == "pg_ctl" && strings.HasSuffix(line, " start") {
+			mu.Lock()
+			storeStarting = true
+			mu.Unlock()
+			time.Sleep(100 * time.Millisecond) // the store takes a while to start
+			defer func() { mu.Lock(); storeUp = true; mu.Unlock() }()
+		}
+		if name == "go" && strings.Contains(line, "-exec true") {
+			time.Sleep(10 * time.Millisecond)
+			mu.Lock()
+			overlap = overlap || (storeStarting && !storeUp)
+			mu.Unlock()
+			for _, e := range env {
+				if strings.HasPrefix(e, "MESH_TEST_") {
+					t.Errorf("the compile was given %s: it runs no test binary", e)
+				}
+			}
+		}
+		return inner(ctx, env, name, args...)
+	}
+	res, st := runLevelTool(t, m, states, "integration")
+	if st.State != delivery.StepPassed {
+		t.Fatalf("the step ended %s: %s", st.State, st.Reason)
+	}
+	if most != 3 {
+		t.Errorf("%d compiles ran at once, want the bound of 3", most)
+	}
+	mu.Lock()
+	defer mu.Unlock()
+	if !overlap {
+		t.Errorf("no compile ran while the store was starting:\n%s", f.lines())
+	}
+	if res.Setup < 0 {
+		t.Errorf("setup = %v", res.Setup)
+	}
+}

modules/mesh-build-go/cmd/mesh-build-go/preview_test.go

@@ -290,8 +290,8 @@ func TestTheIntegrationLevelIsGivenTheBesideDirectoryOnlyWhenSiblingsWerePlaced(
 		}
 		tested := 0
 		for _, c := range f.calls {
-			if !strings.HasPrefix(c.line, "go ") || !strings.Contains(c.line, "-tags integration") {
-				continue
+			if !strings.HasPrefix(c.line, "go ") || !strings.Contains(c.line, "-tags integration") || strings.Contains(c.line, "-exec true") {
+				continue // the setup's compile runs no test binary, so it is given nothing a test reads (issue 1921, S5)
 			}
 			tested++
 			got := ""

modules/mesh-build-go/cmd/mesh-build-go/selected_dirs_test.go

@@ -72,7 +72,7 @@ func TestOnlyTheSelectedDirectoryCompilesAndRuns(t *testing.T) {
 		if !strings.HasPrefix(l, "go -C "+filepath.Join(tree, "b")+" test ") {
 			t.Errorf("go test ran outside the selected directory: %s", l)
 		}
-		if strings.Contains(l, "-run ^$") {
+		if strings.Contains(l, "-exec true") {
 			compiled++
 		} else {
 			ran++
@@ -91,7 +91,7 @@ func TestWithoutASelectionEveryDirectoryCompilesAndRuns(t *testing.T) {
 	compiled, ran := 0, 0
 	for _, l := range strings.Split(f.lines(), "\n") {
 		if strings.HasPrefix(l, "go ") && strings.Contains(l, " test ") {
-			if strings.Contains(l, "-run ^$") {
+			if strings.Contains(l, "-exec true") {
 				compiled++
 			} else {
 				ran++