diff --git a/modules/mesh-build-go/cmd/mesh-build-go/integration.go b/modules/mesh-build-go/cmd/mesh-build-go/integration.go index 231714a0..957514bf 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/integration.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/integration.go @@ -12,6 +12,7 @@ import ( "path/filepath" "strconv" "strings" + "sync" "time" "git.novox.be/novox/mesh-sdk/go/buildstep" @@ -62,30 +63,67 @@ func (m *Module) integrationHere(ctx context.Context, workspace, tree string, pa if err != nil { return nil, err } - postgres, stopStore, err := m.startStore(ctx, run) - if err != nil { - return nil, fmt.Errorf("the integration level's store: %w", err) - } - defer stopStore() - nats, stopBus, err := m.startBus(ctx, filepath.Join(run, "nats")) - if err != nil { - return nil, fmt.Errorf("the integration level's bus: %w", err) - } - defer stopBus() - env := append([]string{"MESH_TEST_POSTGRES=" + postgres, "MESH_TEST_NATS=" + nats}, m.besideEnv(workspace)...) - // Compiling counts as setup: the tests are built once, here, with the addresses a TestMain may read, and the run - // below finds them built. Whether they compile is for the run to say; its failure is not read here. // Only the directories the selected packages lie in compile (ADR 0536 ยง3); no selection is every directory. byDir, err := packagesByDir(tree, dirs, packages) if err != nil { return nil, err } + // The store, the bus and the compile start together (issue 1921, try speed S5); the setup is the longest of them. + // The compile runs no test binary (-exec true), so it needs neither server: whether the tests compile is for the run + // to say, and its failure is not read here; the run below finds them built. + var ( + wg sync.WaitGroup + postgres, nats string + stopStore, stopBus func() + storeErr, busErr error + crash any + crashMu sync.Mutex + ) + guard := func() { + if p := recover(); p != nil { + crashMu.Lock() + crash = p + crashMu.Unlock() + } + } + wg.Add(2) + go func() { defer wg.Done(); defer guard(); postgres, stopStore, storeErr = m.startStore(ctx, run) }() + go func() { + defer wg.Done() + defer guard() + nats, stopBus, busErr = m.startBus(ctx, filepath.Join(run, "nats")) + }() + slots := make(chan struct{}, m.parallel()) for _, d := range dirs { if len(byDir[d]) == 0 { continue } - m.Run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), env...), "go", append([]string{"-C", filepath.Join(tree, d), "test", "-tags", strings.Join(integrationTags, ","), "-run", "^$", "-count=1"}, byDir[d]...)...) + wg.Add(1) + slots <- struct{}{} + go func() { + defer wg.Done() + defer func() { <-slots }() + defer guard() + m.Run(ctx, m.goEnvFor(filepath.Join(tree, d)), "go", append([]string{"-C", filepath.Join(tree, d), "test", "-tags", strings.Join(integrationTags, ","), "-exec", "true", "-count=1"}, byDir[d]...)...) + }() } + wg.Wait() + if stopStore != nil { + defer stopStore() + } + if stopBus != nil { + defer stopBus() + } + if crash != nil { + panic(crash) + } + if storeErr != nil { + return nil, fmt.Errorf("the integration level's store: %w", storeErr) + } + if busErr != nil { + return nil, fmt.Errorf("the integration level's bus: %w", busErr) + } + env := append([]string{"MESH_TEST_POSTGRES=" + postgres, "MESH_TEST_NATS=" + nats}, m.besideEnv(workspace)...) setup := m.clock().Sub(began) return m.runLevel(ctx, tree, levelOpts{ diff --git a/modules/mesh-build-go/cmd/mesh-build-go/integration_level_test.go b/modules/mesh-build-go/cmd/mesh-build-go/integration_level_test.go index fafb7bda..f90ffd66 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/integration_level_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/integration_level_test.go @@ -35,6 +35,8 @@ type integFake struct { // command runs, with its line. gomods string onTest func(line string) + // onCompile is called, unlocked, as each compile of the setup runs (issue 1921, try speed S5). + onCompile func(line string) } func newIntegFake(t *testing.T, captured string) *integFake { @@ -71,7 +73,10 @@ func (f *integFake) run(ctx context.Context, env []string, name string, args ... return forgekey.Ran{Stdout: f.gomods} } return forgekey.Ran{Stdout: "go.mod\n"} - case name == "go" && strings.Contains(line, "-run ^$"): + case name == "go" && strings.Contains(line, "-exec true"): + if f.onCompile != nil { + f.onCompile(line) + } f.tick("compile") case name == "go": if f.onTest != nil { @@ -314,8 +319,8 @@ func TestAStoreThatDoesNotStartEndsTheStepFailedAndLeavesNothing(t *testing.T) { if !strings.Contains(f.lines(), " stop") { t.Errorf("the half-started store was not stopped:\n%s", f.lines()) } - if f.busDir != "" { - t.Errorf("the bus was started after the store failed") + if f.busDir != "" && !f.busDown { + t.Errorf("the bus, started beside the store, was not stopped after the store failed") } if entries, _ := os.ReadDir(filepath.Join(m.Work, "ws1")); len(entries) != 1 || entries[0].Name() != "tree" { t.Errorf("the workspace holds %v, want only its tree", entries) @@ -377,3 +382,68 @@ func TestTheSocketPathStaysUnderTheLimitForALongRunDirectory(t *testing.T) { t.Errorf("the short socket directory %s outlived the store (%v)", sock, err) } } + +// 1921/S5: the setup compiles the go.mod directories side by side, up to the bound, beside the store's and the bus's +// start, and compiles without running a test binary, so it needs neither; the setup stays one number. +func TestTheSetupCompilesTheDirectoriesAtOnceBesideTheStoreAndTheBus(t *testing.T) { + f := newIntegFake(t, "slow") + f.gomods = "a/go.mod\nb/go.mod\nc/go.mod\nd/go.mod\ne/go.mod\n" + var mu sync.Mutex + running, most := 0, 0 + storeStarting, storeUp, overlap := false, false, false + f.onCompile = func(line string) { + mu.Lock() + running++ + most = max(most, running) + mu.Unlock() + time.Sleep(40 * time.Millisecond) + mu.Lock() + running-- + mu.Unlock() + } + m, states := integModule(t, f) + m.Parallel = 3 + tree := filepath.Join(m.Work, "ws1", "tree") + for _, d := range []string{"a", "b", "c", "d", "e"} { + os.MkdirAll(filepath.Join(tree, d), 0o755) + os.WriteFile(filepath.Join(tree, d, "go.mod"), []byte("module x/"+d+"\n"), 0o644) + } + inner := m.Run + m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { + line := strings.Join(args, " ") + if name == "pg_ctl" && strings.HasSuffix(line, " start") { + mu.Lock() + storeStarting = true + mu.Unlock() + time.Sleep(100 * time.Millisecond) // the store takes a while to start + defer func() { mu.Lock(); storeUp = true; mu.Unlock() }() + } + if name == "go" && strings.Contains(line, "-exec true") { + time.Sleep(10 * time.Millisecond) + mu.Lock() + overlap = overlap || (storeStarting && !storeUp) + mu.Unlock() + for _, e := range env { + if strings.HasPrefix(e, "MESH_TEST_") { + t.Errorf("the compile was given %s: it runs no test binary", e) + } + } + } + return inner(ctx, env, name, args...) + } + res, st := runLevelTool(t, m, states, "integration") + if st.State != delivery.StepPassed { + t.Fatalf("the step ended %s: %s", st.State, st.Reason) + } + if most != 3 { + t.Errorf("%d compiles ran at once, want the bound of 3", most) + } + mu.Lock() + defer mu.Unlock() + if !overlap { + t.Errorf("no compile ran while the store was starting:\n%s", f.lines()) + } + if res.Setup < 0 { + t.Errorf("setup = %v", res.Setup) + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/preview_test.go b/modules/mesh-build-go/cmd/mesh-build-go/preview_test.go index 55877c84..7771689a 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/preview_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/preview_test.go @@ -290,8 +290,8 @@ func TestTheIntegrationLevelIsGivenTheBesideDirectoryOnlyWhenSiblingsWerePlaced( } tested := 0 for _, c := range f.calls { - if !strings.HasPrefix(c.line, "go ") || !strings.Contains(c.line, "-tags integration") { - continue + if !strings.HasPrefix(c.line, "go ") || !strings.Contains(c.line, "-tags integration") || strings.Contains(c.line, "-exec true") { + continue // the setup's compile runs no test binary, so it is given nothing a test reads (issue 1921, S5) } tested++ got := "" diff --git a/modules/mesh-build-go/cmd/mesh-build-go/selected_dirs_test.go b/modules/mesh-build-go/cmd/mesh-build-go/selected_dirs_test.go index fa86bb54..095ff793 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/selected_dirs_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/selected_dirs_test.go @@ -72,7 +72,7 @@ func TestOnlyTheSelectedDirectoryCompilesAndRuns(t *testing.T) { if !strings.HasPrefix(l, "go -C "+filepath.Join(tree, "b")+" test ") { t.Errorf("go test ran outside the selected directory: %s", l) } - if strings.Contains(l, "-run ^$") { + if strings.Contains(l, "-exec true") { compiled++ } else { ran++ @@ -91,7 +91,7 @@ func TestWithoutASelectionEveryDirectoryCompilesAndRuns(t *testing.T) { compiled, ran := 0, 0 for _, l := range strings.Split(f.lines(), "\n") { if strings.HasPrefix(l, "go ") && strings.Contains(l, " test ") { - if strings.Contains(l, "-run ^$") { + if strings.Contains(l, "-exec true") { compiled++ } else { ran++