diff --git a/modules/mesh-build-go/cmd/mesh-build-go/integration.go b/modules/mesh-build-go/cmd/mesh-build-go/integration.go index 03de39ab..231714a0 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/integration.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/integration.go @@ -75,8 +75,16 @@ func (m *Module) integrationHere(ctx context.Context, workspace, tree string, pa env := append([]string{"MESH_TEST_POSTGRES=" + postgres, "MESH_TEST_NATS=" + nats}, m.besideEnv(workspace)...) // Compiling counts as setup: the tests are built once, here, with the addresses a TestMain may read, and the run // below finds them built. Whether they compile is for the run to say; its failure is not read here. + // Only the directories the selected packages lie in compile (ADR 0536 §3); no selection is every directory. + byDir, err := packagesByDir(tree, dirs, packages) + if err != nil { + return nil, err + } for _, d := range dirs { - m.Run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), env...), "go", "-C", filepath.Join(tree, d), "test", "-tags", strings.Join(integrationTags, ","), "-run", "^$", "-count=1", "./...") + if len(byDir[d]) == 0 { + continue + } + m.Run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), env...), "go", append([]string{"-C", filepath.Join(tree, d), "test", "-tags", strings.Join(integrationTags, ","), "-run", "^$", "-count=1"}, byDir[d]...)...) } setup := m.clock().Sub(began) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/integration_level_test.go b/modules/mesh-build-go/cmd/mesh-build-go/integration_level_test.go index 9d2aafa1..fafb7bda 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/integration_level_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/integration_level_test.go @@ -31,6 +31,10 @@ type integFake struct { busDown bool runDirs []string // run directories seen when the store was started, to be gone at the end panicOn string // a phase that panics + // gomods is what git ls-files answers ("go.mod\n" when empty); onTest is called, unlocked, as each go test -json + // command runs, with its line. + gomods string + onTest func(line string) } func newIntegFake(t *testing.T, captured string) *integFake { @@ -63,10 +67,16 @@ func (f *integFake) run(ctx context.Context, env []string, name string, args ... case name == "postgres": return forgekey.Ran{Stdout: f.postgres + "\n"} case name == "git" && len(args) > 2 && args[2] == "ls-files": + if f.gomods != "" { + return forgekey.Ran{Stdout: f.gomods} + } return forgekey.Ran{Stdout: "go.mod\n"} case name == "go" && strings.Contains(line, "-run ^$"): f.tick("compile") case name == "go": + if f.onTest != nil { + f.onTest(line) + } f.tick("test") status := 0 if strings.Contains(f.captured, `"Action":"fail"`) { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/module.go b/modules/mesh-build-go/cmd/mesh-build-go/module.go index fffb3e1f..0f306017 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/module.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/module.go @@ -54,6 +54,9 @@ type Module struct { // Bubblewrap reads the installed bubblewrap's version (ProbeBubblewrap): a step is refused, and the module is // unhealthy, while it is missing or older than BubblewrapLeast. Nil probes nothing. Bubblewrap func(ctx context.Context) (string, error) + // Parallel is how many go.mod directories a level runs at once; zero is half the processors, and at least one + // (ADR 0536 §3: a node runs two checks at once). + Parallel int // Inside is the integration level's own process, started shut off in bubblewrap by a test step (level.go): its // commands run as they are, never shut off a second time. Inside bool diff --git a/modules/mesh-build-go/cmd/mesh-build-go/selected_dirs_test.go b/modules/mesh-build-go/cmd/mesh-build-go/selected_dirs_test.go new file mode 100644 index 00000000..fa86bb54 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/selected_dirs_test.go @@ -0,0 +1,172 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// threeModules gives the workspace three go.mod directories, a, b and c, whose modules are fixture/a, fixture/b and +// fixture/c. +func threeModules(t *testing.T, f *integFake) (*Module, *keptStates) { + t.Helper() + m, states := integModule(t, f) + f.gomods = "a/go.mod\nb/go.mod\nc/go.mod\n" + for _, d := range []string{"a", "b", "c"} { + dir := filepath.Join(m.Work, "ws1", "tree", d) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "go.mod"), []byte("module fixture/"+d+"\n\ngo 1.25.0\n"), 0o644); err != nil { + t.Fatal(err) + } + } + return m, states +} + +func runIntegrationOf(t *testing.T, m *Module, states *keptStates, packages ...any) (TestResult, delivery.Step) { + t.Helper() + for _, tool := range m.Tools(m.StepTools()) { + if tool.Name != "build_go_test" { + continue + } + args := map[string]any{"workspace": "ws1", "level": "integration"} + if len(packages) > 0 { + args["packages"] = packages + } + if _, err := tool.Run(args); err != nil { + t.Fatal(err) + } + } + st := states.waitEnd(t) + var res TestResult + if st.State == delivery.StepPassed { + if err := json.Unmarshal(st.Result, &res); err != nil { + t.Fatal(err) + } + } + return res, st +} + +// Issue 890/AC1 (ADR 0536 §3): with packages selected in one of three go.mod directories, the integration level +// compiles and runs go test in that directory only. +func TestOnlyTheSelectedDirectoryCompilesAndRuns(t *testing.T) { + f := newIntegFake(t, "slow") + m, states := threeModules(t, f) + if _, st := runIntegrationOf(t, m, states, "fixture/b/x"); st.State != delivery.StepPassed { + t.Fatalf("the step ended %s: %s", st.State, st.Reason) + } + tree := filepath.Join(m.Work, "ws1", "tree") + compiled, ran := 0, 0 + for _, l := range strings.Split(f.lines(), "\n") { + if !strings.HasPrefix(l, "go ") || !strings.Contains(l, " test ") { + continue + } + if !strings.HasPrefix(l, "go -C "+filepath.Join(tree, "b")+" test ") { + t.Errorf("go test ran outside the selected directory: %s", l) + } + if strings.Contains(l, "-run ^$") { + compiled++ + } else { + ran++ + } + } + if compiled != 1 || ran != 1 { + t.Errorf("compiled %d times and ran %d times, want once each:\n%s", compiled, ran, f.lines()) + } +} + +// Issue 890/AC1: no selection keeps every directory, compiled and run. +func TestWithoutASelectionEveryDirectoryCompilesAndRuns(t *testing.T) { + f := newIntegFake(t, "slow") + m, states := threeModules(t, f) + runIntegrationOf(t, m, states) + compiled, ran := 0, 0 + for _, l := range strings.Split(f.lines(), "\n") { + if strings.HasPrefix(l, "go ") && strings.Contains(l, " test ") { + if strings.Contains(l, "-run ^$") { + compiled++ + } else { + ran++ + } + } + } + if compiled != 3 || ran != 3 { + t.Errorf("compiled %d and ran %d, want 3 and 3:\n%s", compiled, ran, f.lines()) + } +} + +// Issue 890/AC2: with two directories selected and a runner that blocks, both go test commands run at once; the +// result lists the packages in directory order. +func TestSelectedDirectoriesRunAtOnceAndAnswerInDirectoryOrder(t *testing.T) { + f := newIntegFake(t, "slow") + var mu sync.Mutex + running, most := 0, 0 + release := make(chan struct{}) + f.onTest = func(line string) { + mu.Lock() + running++ + if running > most { + most = running + } + if running == 2 { + close(release) + } + mu.Unlock() + select { + case <-release: + case <-time.After(10 * time.Second): + } + mu.Lock() + running-- + mu.Unlock() + } + m, states := threeModules(t, f) + m.Parallel = 3 + res, st := runIntegrationOf(t, m, states, "fixture/c/x", "fixture/a/x") + if st.State != delivery.StepPassed { + t.Fatalf("the step ended %s: %s", st.State, st.Reason) + } + if most != 2 { + t.Errorf("at most %d go test commands ran at once, want 2", most) + } + dirs := []string{} + for _, p := range res.Packages { + dirs = append(dirs, p.Dir) + } + if got := fmt.Sprint(dirs); !strings.HasPrefix(got, "[a ") || !strings.HasSuffix(got, " c]") { + t.Errorf("the packages lie in directories %s, want a's before c's", got) + } +} + +// The bound holds: with one directory at a time, no two go test commands run at once. +func TestTheDirectoriesAtOnceNeverPassTheBound(t *testing.T) { + f := newIntegFake(t, "slow") + var mu sync.Mutex + running, most := 0, 0 + f.onTest = func(string) { + mu.Lock() + running++ + if running > most { + most = running + } + mu.Unlock() + time.Sleep(20 * time.Millisecond) + mu.Lock() + running-- + mu.Unlock() + } + m, states := threeModules(t, f) + m.Parallel = 1 + runIntegrationOf(t, m, states) + if most != 1 { + t.Errorf("%d go test commands ran at once with a bound of 1", most) + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/test.go b/modules/mesh-build-go/cmd/mesh-build-go/test.go index d129c277..5add8055 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/test.go @@ -10,8 +10,12 @@ import ( "os" "path/filepath" "regexp" + "runtime" "strings" + "sync" "time" + + "git.novox.be/novox/mesh-sdk/go/forgekey" ) // testTimeout is go test's own timeout for a level: a bound on a test that never ends, far above any level's bound. @@ -143,26 +147,71 @@ func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestR return nil, err } began := m.clock() + // The directories that hold a selected package run side by side, up to the bound, each into a result of its own; + // the results join in directory order, so the answer does not depend on which ended first (ADR 0536 §3). + type dirRun struct { + res *TestResult + rd *reader + r runAnswer + noise []string + } + var todo []string for _, d := range dirs { - packages := byDir[d] - if len(packages) == 0 { - continue // no package asked for lies in this module + if len(byDir[d]) > 0 { // no package asked for lies in the others + todo = append(todo, d) } - args := []string{"-C", filepath.Join(tree, d), "test", "-json", "-count=1", "-timeout", testTimeout} - if len(o.tags) > 0 { - args = append(args, "-tags", strings.Join(o.tags, ",")) - } - if o.run != "" { - args = append(args, "-run", o.run) - } - args = append(args, packages...) - r := run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), o.env...), "go", args...) - rd := newReader(res, d) - noise := rd.read(r.Stdout) - rd.finish() - if (r.Status != 0 || r.Err != "") && len(rd.pkgs) == 0 { + } + runs := make([]dirRun, len(todo)) + slots := make(chan struct{}, m.parallel()) + var wg sync.WaitGroup + var crash any // a panic of a run goes on in the caller, which cleans up after it + var crashMu sync.Mutex + for i, d := range todo { + wg.Add(1) + slots <- struct{}{} + go func() { + defer wg.Done() + defer func() { <-slots }() + defer func() { + if p := recover(); p != nil { + crashMu.Lock() + crash = p + crashMu.Unlock() + } + }() + args := []string{"-C", filepath.Join(tree, d), "test", "-json", "-count=1", "-timeout", testTimeout} + if len(o.tags) > 0 { + args = append(args, "-tags", strings.Join(o.tags, ",")) + } + if o.run != "" { + args = append(args, "-run", o.run) + } + args = append(args, byDir[d]...) + dr := &runs[i] + dr.res = &TestResult{} + dr.r = run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), o.env...), "go", args...) + dr.rd = newReader(dr.res, d) + dr.noise = dr.rd.read(dr.r.Stdout) + dr.rd.finish() + }() + } + wg.Wait() + if crash != nil { + panic(crash) + } + for i, d := range todo { + dr := runs[i] + if (dr.r.Status != 0 || dr.r.Err != "") && len(dr.rd.pkgs) == 0 { // go test said nothing a test said: it could not run (a go.mod it cannot read, a toolchain it cannot find). - return nil, fmt.Errorf("go test in %s: %s", d, firstLine(r.Err+"\n"+strings.Join(noise, "\n")+"\n"+r.Stderr)) + return nil, fmt.Errorf("go test in %s: %s", d, firstLine(dr.r.Err+"\n"+strings.Join(dr.noise, "\n")+"\n"+dr.r.Stderr)) + } + res.Tests += dr.res.Tests + res.Skipped += dr.res.Skipped + res.Failed = append(res.Failed, dr.res.Failed...) + res.Packages = append(res.Packages, dr.res.Packages...) + res.Slowest = append(res.Slowest, dr.res.Slowest...) + if dr.res.Elapsed > res.Elapsed { + res.Elapsed = dr.res.Elapsed } } if o.wall { @@ -172,6 +221,17 @@ func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestR return res, nil } +// runAnswer is what a step runner answers. +type runAnswer = forgekey.Ran + +// parallel is how many directories a level runs at once. +func (m *Module) parallel() int { + if m.Parallel > 0 { + return m.Parallel + } + return max(1, runtime.NumCPU()/2) +} + // packagesByDir says which patterns run in which go.mod directory. A relative pattern (./...) runs in every directory, as // it always did; an import path, as build_go_digest keys its packages, runs only in the directory whose module holds it // (the longest module path that is the path or its prefix), as another module's path would fail the step. No pattern