novox/mesh-catalog
18
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases Wiki Activity

Compile and run only the go.mod directories a selection lies in, and run them side by side up to a bound (issue 890, ADR 0536 s3)

Browse Source
This commit is contained in:
jschoubben
2026-10-23 14:04:43 +02:00
parent 47dbf87e33
commit 547f4ee27d
5 changed files with 271 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/mesh-build-go/cmd/mesh-build-go/integration.go
+9 -1
View File
@@ -75,8 +75,16 @@ func (m *Module) integrationHere(ctx context.Context, workspace, tree string, pa
env := append([]string{"MESH_TEST_POSTGRES=" + postgres, "MESH_TEST_NATS=" + nats}, m.besideEnv(workspace)...)
// Compiling counts as setup: the tests are built once, here, with the addresses a TestMain may read, and the run
// below finds them built. Whether they compile is for the run to say; its failure is not read here.
// Only the directories the selected packages lie in compile (ADR 0536 §3); no selection is every directory.
byDir, err := packagesByDir(tree, dirs, packages)
if err != nil {
return nil, err
}
for _, d := range dirs {
m.Run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), env...), "go", "-C", filepath.Join(tree, d), "test", "-tags", strings.Join(integrationTags, ","), "-run", "^$", "-count=1", "./...")
if len(byDir[d]) == 0 {
continue
}
m.Run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), env...), "go", append([]string{"-C", filepath.Join(tree, d), "test", "-tags", strings.Join(integrationTags, ","), "-run", "^$", "-count=1"}, byDir[d]...)...)
}
setup := m.clock().Sub(began)
modules/mesh-build-go/cmd/mesh-build-go/integration_level_test.go
+10
View File
@@ -31,6 +31,10 @@ type integFake struct {
busDown bool
runDirs []string // run directories seen when the store was started, to be gone at the end
panicOn string // a phase that panics
// gomods is what git ls-files answers ("go.mod\n" when empty); onTest is called, unlocked, as each go test -json
// command runs, with its line.
gomods string
onTest func(line string)
}
func newIntegFake(t *testing.T, captured string) *integFake {
@@ -63,10 +67,16 @@ func (f *integFake) run(ctx context.Context, env []string, name string, args ...
case name == "postgres":
return forgekey.Ran{Stdout: f.postgres + "\n"}
case name == "git" && len(args) > 2 && args[2] == "ls-files":
if f.gomods != "" {
return forgekey.Ran{Stdout: f.gomods}
}
return forgekey.Ran{Stdout: "go.mod\n"}
case name == "go" && strings.Contains(line, "-run ^$"):
f.tick("compile")
case name == "go":
if f.onTest != nil {
f.onTest(line)
}
f.tick("test")
status := 0
if strings.Contains(f.captured, `"Action":"fail"`) {
modules/mesh-build-go/cmd/mesh-build-go/module.go
+3
View File
@@ -54,6 +54,9 @@ type Module struct {
// Bubblewrap reads the installed bubblewrap's version (ProbeBubblewrap): a step is refused, and the module is
// unhealthy, while it is missing or older than BubblewrapLeast. Nil probes nothing.
Bubblewrap func(ctx context.Context) (string, error)
// Parallel is how many go.mod directories a level runs at once; zero is half the processors, and at least one
// (ADR 0536 §3: a node runs two checks at once).
Parallel int
// Inside is the integration level's own process, started shut off in bubblewrap by a test step (level.go): its
// commands run as they are, never shut off a second time.
Inside bool
modules/mesh-build-go/cmd/mesh-build-go/selected_dirs_test.go
+172
View File
@@ -0,0 +1,172 @@
package main
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// threeModules gives the workspace three go.mod directories, a, b and c, whose modules are fixture/a, fixture/b and
// fixture/c.
func threeModules(t *testing.T, f *integFake) (*Module, *keptStates) {
t.Helper()
m, states := integModule(t, f)
f.gomods = "a/go.mod\nb/go.mod\nc/go.mod\n"
for _, d := range []string{"a", "b", "c"} {
dir := filepath.Join(m.Work, "ws1", "tree", d)
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "go.mod"), []byte("module fixture/"+d+"\n\ngo 1.25.0\n"), 0o644); err != nil {
t.Fatal(err)
}
}
return m, states
}
func runIntegrationOf(t *testing.T, m *Module, states *keptStates, packages ...any) (TestResult, delivery.Step) {
t.Helper()
for _, tool := range m.Tools(m.StepTools()) {
if tool.Name != "build_go_test" {
continue
}
args := map[string]any{"workspace": "ws1", "level": "integration"}
if len(packages) > 0 {
args["packages"] = packages
}
if _, err := tool.Run(args); err != nil {
t.Fatal(err)
}
}
st := states.waitEnd(t)
var res TestResult
if st.State == delivery.StepPassed {
if err := json.Unmarshal(st.Result, &res); err != nil {
t.Fatal(err)
}
}
return res, st
}
// Issue 890/AC1 (ADR 0536 §3): with packages selected in one of three go.mod directories, the integration level
// compiles and runs go test in that directory only.
func TestOnlyTheSelectedDirectoryCompilesAndRuns(t *testing.T) {
f := newIntegFake(t, "slow")
m, states := threeModules(t, f)
if _, st := runIntegrationOf(t, m, states, "fixture/b/x"); st.State != delivery.StepPassed {
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
}
tree := filepath.Join(m.Work, "ws1", "tree")
compiled, ran := 0, 0
for _, l := range strings.Split(f.lines(), "\n") {
if !strings.HasPrefix(l, "go ") || !strings.Contains(l, " test ") {
continue
}
if !strings.HasPrefix(l, "go -C "+filepath.Join(tree, "b")+" test ") {
t.Errorf("go test ran outside the selected directory: %s", l)
}
if strings.Contains(l, "-run ^$") {
compiled++
} else {
ran++
}
}
if compiled != 1 || ran != 1 {
t.Errorf("compiled %d times and ran %d times, want once each:\n%s", compiled, ran, f.lines())
}
}
// Issue 890/AC1: no selection keeps every directory, compiled and run.
func TestWithoutASelectionEveryDirectoryCompilesAndRuns(t *testing.T) {
f := newIntegFake(t, "slow")
m, states := threeModules(t, f)
runIntegrationOf(t, m, states)
compiled, ran := 0, 0
for _, l := range strings.Split(f.lines(), "\n") {
if strings.HasPrefix(l, "go ") && strings.Contains(l, " test ") {
if strings.Contains(l, "-run ^$") {
compiled++
} else {
ran++
}
}
}
if compiled != 3 || ran != 3 {
t.Errorf("compiled %d and ran %d, want 3 and 3:\n%s", compiled, ran, f.lines())
}
}
// Issue 890/AC2: with two directories selected and a runner that blocks, both go test commands run at once; the
// result lists the packages in directory order.
func TestSelectedDirectoriesRunAtOnceAndAnswerInDirectoryOrder(t *testing.T) {
f := newIntegFake(t, "slow")
var mu sync.Mutex
running, most := 0, 0
release := make(chan struct{})
f.onTest = func(line string) {
mu.Lock()
running++
if running > most {
most = running
}
if running == 2 {
close(release)
}
mu.Unlock()
select {
case <-release:
case <-time.After(10 * time.Second):
}
mu.Lock()
running--
mu.Unlock()
}
m, states := threeModules(t, f)
m.Parallel = 3
res, st := runIntegrationOf(t, m, states, "fixture/c/x", "fixture/a/x")
if st.State != delivery.StepPassed {
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
}
if most != 2 {
t.Errorf("at most %d go test commands ran at once, want 2", most)
}
dirs := []string{}
for _, p := range res.Packages {
dirs = append(dirs, p.Dir)
}
if got := fmt.Sprint(dirs); !strings.HasPrefix(got, "[a ") || !strings.HasSuffix(got, " c]") {
t.Errorf("the packages lie in directories %s, want a's before c's", got)
}
}
// The bound holds: with one directory at a time, no two go test commands run at once.
func TestTheDirectoriesAtOnceNeverPassTheBound(t *testing.T) {
f := newIntegFake(t, "slow")
var mu sync.Mutex
running, most := 0, 0
f.onTest = func(string) {
mu.Lock()
running++
if running > most {
most = running
}
mu.Unlock()
time.Sleep(20 * time.Millisecond)
mu.Lock()
running--
mu.Unlock()
}
m, states := threeModules(t, f)
m.Parallel = 1
runIntegrationOf(t, m, states)
if most != 1 {
t.Errorf("%d go test commands ran at once with a bound of 1", most)
}
}
modules/mesh-build-go/cmd/mesh-build-go/test.go
+77 -17
View File
@@ -10,8 +10,12 @@ import (
"os"
"path/filepath"
"regexp"
"runtime"
"strings"
"sync"
"time"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
// testTimeout is go test's own timeout for a level: a bound on a test that never ends, far above any level's bound.
@@ -143,26 +147,71 @@ func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestR
return nil, err
}
began := m.clock()
// The directories that hold a selected package run side by side, up to the bound, each into a result of its own;
// the results join in directory order, so the answer does not depend on which ended first (ADR 0536 §3).
type dirRun struct {
res *TestResult
rd *reader
r runAnswer
noise []string
}
var todo []string
for _, d := range dirs {
packages := byDir[d]
if len(packages) == 0 {
continue // no package asked for lies in this module
if len(byDir[d]) > 0 { // no package asked for lies in the others
todo = append(todo, d)
}
args := []string{"-C", filepath.Join(tree, d), "test", "-json", "-count=1", "-timeout", testTimeout}
if len(o.tags) > 0 {
args = append(args, "-tags", strings.Join(o.tags, ","))
}
if o.run != "" {
args = append(args, "-run", o.run)
}
args = append(args, packages...)
r := run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), o.env...), "go", args...)
rd := newReader(res, d)
noise := rd.read(r.Stdout)
rd.finish()
if (r.Status != 0 || r.Err != "") && len(rd.pkgs) == 0 {
}
runs := make([]dirRun, len(todo))
slots := make(chan struct{}, m.parallel())
var wg sync.WaitGroup
var crash any // a panic of a run goes on in the caller, which cleans up after it
var crashMu sync.Mutex
for i, d := range todo {
wg.Add(1)
slots <- struct{}{}
go func() {
defer wg.Done()
defer func() { <-slots }()
defer func() {
if p := recover(); p != nil {
crashMu.Lock()
crash = p
crashMu.Unlock()
}
}()
args := []string{"-C", filepath.Join(tree, d), "test", "-json", "-count=1", "-timeout", testTimeout}
if len(o.tags) > 0 {
args = append(args, "-tags", strings.Join(o.tags, ","))
}
if o.run != "" {
args = append(args, "-run", o.run)
}
args = append(args, byDir[d]...)
dr := &runs[i]
dr.res = &TestResult{}
dr.r = run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), o.env...), "go", args...)
dr.rd = newReader(dr.res, d)
dr.noise = dr.rd.read(dr.r.Stdout)
dr.rd.finish()
}()
}
wg.Wait()
if crash != nil {
panic(crash)
}
for i, d := range todo {
dr := runs[i]
if (dr.r.Status != 0 || dr.r.Err != "") && len(dr.rd.pkgs) == 0 {
// go test said nothing a test said: it could not run (a go.mod it cannot read, a toolchain it cannot find).
return nil, fmt.Errorf("go test in %s: %s", d, firstLine(r.Err+"\n"+strings.Join(noise, "\n")+"\n"+r.Stderr))
return nil, fmt.Errorf("go test in %s: %s", d, firstLine(dr.r.Err+"\n"+strings.Join(dr.noise, "\n")+"\n"+dr.r.Stderr))
}
res.Tests += dr.res.Tests
res.Skipped += dr.res.Skipped
res.Failed = append(res.Failed, dr.res.Failed...)
res.Packages = append(res.Packages, dr.res.Packages...)
res.Slowest = append(res.Slowest, dr.res.Slowest...)
if dr.res.Elapsed > res.Elapsed {
res.Elapsed = dr.res.Elapsed
}
}
if o.wall {
@@ -172,6 +221,17 @@ func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestR
return res, nil
}
// runAnswer is what a step runner answers.
type runAnswer = forgekey.Ran
// parallel is how many directories a level runs at once.
func (m *Module) parallel() int {
if m.Parallel > 0 {
return m.Parallel
}
return max(1, runtime.NumCPU()/2)
}
// packagesByDir says which patterns run in which go.mod directory. A relative pattern (./...) runs in every directory, as
// it always did; an import path, as build_go_digest keys its packages, runs only in the directory whose module holds it
// (the longest module path that is the path or its prefix), as another module's path would fail the step. No pattern