mesh-catalog/modules/mesh-build-go/cmd/mesh-build-go/test.go

package main

// The test step (novox/hq ADR 0445; issues 1290 and 1486): a fetched workspace's tests by Go's convention, one level
// at a time, each judged by its own bounds. The unit level is `go test` over every package without the `integration`
// build tag.

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
	"sync"
	"time"

	"git.novox.be/novox/mesh-sdk/go/forgekey"
)

// testTimeout is go test's own timeout for a level: a bound on a test that never ends, far above any level's bound.
const testTimeout = "10m"

var packageOK = regexp.MustCompile(`^(\./|[A-Za-z0-9])[A-Za-z0-9._/@~-]*(/\.\.\.)?$`)

func (m *Module) testTool() StepTool {
	return StepTool{
		Name: "build_go_test",
		Description: "Run the tests of a fetched workspace by Go's convention, one level: the unit level is go test over every " +
			"package without the integration build tag, within five seconds. Answers a step id at once; the step ends on " +
			"its key with the level's result: its packages, its times, the tests that failed and the slowest tests. A " +
			"level that fails is a finished step whose result says passed false.",
		Input: map[string]any{
			"type": "object",
			"properties": map[string]any{
				"workspace": map[string]any{"type": "string", "description": "the workspace id build_go_fetch answered"},
				"level":     map[string]any{"type": "string", "enum": []string{LevelUnit, LevelIntegration}, "description": "the level"},
				"packages": map[string]any{"type": "array", "items": map[string]any{"type": "string"},
					"description": "package patterns of each module, ./... when none"},
				"run":    map[string]any{"type": "string", "description": "optional go test -run pattern: only the tests it matches run (a try's tests)"},
				"runner": runnerProperty,
			},
			"required": []string{"workspace", "level"},
		},
		Long: true,
		Run:  m.test,
	}
}

// unitBound is the unit level's bound: UnitBound unless a test lowered it.
func (m *Module) unitBound() time.Duration {
	if m.UnitBound > 0 {
		return m.UnitBound
	}
	return UnitBound
}

func (m *Module) test(ctx context.Context, args map[string]any) (any, error) {
	workspace, _ := args["workspace"].(string)
	level, _ := args["level"].(string)
	packages, err := packagesArg(args["packages"])
	if err != nil {
		return nil, err
	}
	pattern, _ := args["run"].(string)
	if !runPatternOK(pattern) {
		return nil, fmt.Errorf("%q is not a go test -run pattern", pattern)
	}
	if !workspaceOK.MatchString(workspace) {
		return nil, fmt.Errorf("%q is not a workspace id", workspace)
	}
	tree := filepath.Join(m.Work, workspace, "tree")
	if _, err := os.Stat(tree); err != nil {
		return nil, fmt.Errorf("no workspace %s: it was never fetched here, or it is older than a day and was removed", workspace)
	}
	runner, err := runnerArg(args["runner"])
	if err != nil {
		return nil, err
	}
	var res *TestResult
	switch {
	case runner == RunnerWorkspace && (level == LevelUnit || level == LevelIntegration):
		return m.testAsCandidate(ctx, workspace, level, packages, pattern)
	case level == LevelUnit:
		res, err = m.runLevel(ctx, tree, levelOpts{workspace: workspace, level: LevelUnit, packages: packages, run: pattern, bounds: Bounds{Run: m.unitBound()}, env: m.besideEnv(workspace)})
	case level == LevelIntegration:
		res, err = m.integration(ctx, workspace, tree, packages, pattern)
	default:
		return nil, fmt.Errorf("the level %q is neither unit nor integration", level)
	}
	if err != nil {
		return nil, err
	}
	res.JudgedBy = RunnerInstalled
	return res, nil
}

// testAsCandidate runs a level as the candidate's program (ADR 0535 §1): the integration level is refused first as the
// installed program would refuse it (no postgres), then the whole level runs in the candidate's one shut-off process.
func (m *Module) testAsCandidate(ctx context.Context, workspace, level string, packages []string, pattern string) (*TestResult, error) {
	spec := levelSpec{Step: stepUnit, Workspace: workspace, Packages: packages, Run: pattern, Bound: m.unitBound()}
	if level == LevelIntegration {
		if _, err := m.checkPostgres(ctx); err != nil {
			return nil, err // a *buildstep.Refusal: this level only
		}
		b := m.integrationBounds()
		spec = levelSpec{Step: stepIntegration, Workspace: workspace, Packages: packages, Run: pattern, Setup: b.Setup, Bound: b.Run}
	}
	a, judged, err := m.asCandidate(ctx, spec)
	if err != nil {
		return nil, err
	}
	if a.Result == nil {
		return nil, fmt.Errorf("the candidate's mesh-build-go answered no result of the %s level", level)
	}
	a.Result.JudgedBy = judged
	return a.Result, nil
}

// besideEnv is MESH_CHECK_BESIDE, naming the directory build_go_fetch placed the workspace's siblings in (ADR 0484
// clause 4), when it placed any; the tests that read other repositories look there (mesh-sdk go/internal/beside).
func (m *Module) besideEnv(workspace string) []string {
	dir := m.besideDir(workspace)
	if st, err := os.Stat(dir); err != nil || !st.IsDir() {
		return nil
	}
	return []string{"MESH_CHECK_BESIDE=" + dir}
}

// packagesArg reads the packages argument: a list of package patterns, as []any when it came over the bus as JSON and
// as []string when an ask handed it in process (issue 1925).
func packagesArg(v any) ([]string, error) {
	var list []string
	switch t := v.(type) {
	case nil:
		return nil, nil
	case []string:
		list = t
	case []any:
		for _, e := range t {
			s, _ := e.(string)
			list = append(list, s)
		}
	default:
		return nil, fmt.Errorf("packages is a list of package patterns")
	}
	out := []string{}
	for _, s := range list {
		if !packageOK.MatchString(s) || strings.Contains(strings.TrimSuffix(s, "/..."), "..") {
			return nil, fmt.Errorf("%q is not a package pattern", s)
		}
		out = append(out, s)
	}
	return out, nil
}

// levelOpts are what a level asks of runLevel.
type levelOpts struct {
	// workspace is the workspace's id: its commands run shut off in it (stepRunner).
	workspace string
	// run is go test's -run pattern, "" for every test.
	run      string
	level    string
	tags     []string
	packages []string
	bounds   Bounds
	// env is what the level adds to a go command's environment (MESH_TEST_POSTGRES, MESH_TEST_NATS).
	env []string
	// setup is the time the level's setup took, which the caller measured; wall says the level's running time is the
	// wall time of its go test commands, as the integration level's packages run side by side by go test's own
	// parallelism, where the unit level's is the slowest package's.
	setup time.Duration
	wall  bool
	// compose is a state of the level to put in the result (the store and the bus it ran against).
	store, bus string
}

// runLevel runs `go test -json` in each go.mod directory of the tree and judges the whole against the bounds.
func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestResult, error) {
	dirs, err := m.goModDirs(ctx, tree)
	if err != nil {
		return nil, err
	}
	res := &TestResult{Level: o.level, Packages: []PackageResult{}, Setup: o.setup.Seconds(), Store: o.store, Bus: o.bus}
	byDir, err := packagesByDir(tree, dirs, o.packages)
	if err != nil {
		return nil, err
	}
	run, err := m.stepRunner(o.workspace, false)
	if err != nil {
		return nil, err
	}
	began := m.clock()
	// The directories that hold a selected package run side by side, up to the bound, each into a result of its own;
	// the results join in directory order, so the answer does not depend on which ended first (ADR 0536 §3).
	type dirRun struct {
		res   *TestResult
		rd    *reader
		r     runAnswer
		noise []string
	}
	var todo []string
	for _, d := range dirs {
		if len(byDir[d]) > 0 { // no package asked for lies in the others
			todo = append(todo, d)
		}
	}
	if o.level == LevelUnit {
		// gofmt and go vet of each directory the level runs come before its tests (ADR 0529 §1; issue 1887 D7-go): a
		// directory that fails them fails the level at once, and their time is the result's Lint.
		linted := m.clock()
		failed, err := m.lintAll(ctx, run, tree, todo, func(d string) []string {
			return append(m.goEnvFor(filepath.Join(tree, d)), o.env...)
		})
		if err != nil {
			return nil, err
		}
		res.Lint = m.clock().Sub(linted).Seconds()
		if len(failed) > 0 {
			res.Failed = failed
			res.judge(o.bounds)
			return res, nil
		}
		began = m.clock() // the level's running time is its tests'
	}
	runs := make([]dirRun, len(todo))
	slots := make(chan struct{}, m.parallel())
	var wg sync.WaitGroup
	var crash any // a panic of a run goes on in the caller, which cleans up after it
	var crashMu sync.Mutex
	for i, d := range todo {
		wg.Add(1)
		slots <- struct{}{}
		go func() {
			defer wg.Done()
			defer func() { <-slots }()
			defer func() {
				if p := recover(); p != nil {
					crashMu.Lock()
					crash = p
					crashMu.Unlock()
				}
			}()
			args := []string{"-C", filepath.Join(tree, d), "test", "-json", "-count=1", "-timeout", testTimeout}
			if len(o.tags) > 0 {
				args = append(args, "-tags", strings.Join(o.tags, ","))
			}
			if o.run != "" {
				args = append(args, "-run", o.run)
			}
			args = append(args, byDir[d]...)
			dr := &runs[i]
			dr.res = &TestResult{}
			dr.r = run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), o.env...), "go", args...)
			dr.rd = newReader(dr.res, d)
			dr.noise = dr.rd.read(dr.r.Stdout)
			dr.rd.finish()
		}()
	}
	wg.Wait()
	if crash != nil {
		panic(crash)
	}
	for i, d := range todo {
		dr := runs[i]
		if (dr.r.Status != 0 || dr.r.Err != "") && len(dr.rd.pkgs) == 0 {
			// go test said nothing a test said: it could not run (a go.mod it cannot read, a toolchain it cannot find).
			return nil, fmt.Errorf("go test in %s: %s", d, firstLine(dr.r.Err+"\n"+strings.Join(dr.noise, "\n")+"\n"+dr.r.Stderr))
		}
		res.Tests += dr.res.Tests
		res.Skipped += dr.res.Skipped
		res.Failed = append(res.Failed, dr.res.Failed...)
		res.Packages = append(res.Packages, dr.res.Packages...)
		res.Slowest = append(res.Slowest, dr.res.Slowest...)
		if dr.res.Elapsed > res.Elapsed {
			res.Elapsed = dr.res.Elapsed
		}
	}
	if o.wall {
		res.Elapsed = m.clock().Sub(began).Seconds()
	}
	res.judge(o.bounds)
	return res, nil
}

// runAnswer is what a step runner answers.
type runAnswer = forgekey.Ran

// parallel is how many directories a level runs at once.
func (m *Module) parallel() int {
	if m.Parallel > 0 {
		return m.Parallel
	}
	return max(1, runtime.NumCPU()/2)
}

// packagesByDir says which patterns run in which go.mod directory. A relative pattern (./...) runs in every directory, as
// it always did; an import path, as build_go_digest keys its packages, runs only in the directory whose module holds it
// (the longest module path that is the path or its prefix), as another module's path would fail the step. No pattern
// means ./... everywhere. An import path no module of the tree holds is an error that names it.
func packagesByDir(tree string, dirs, patterns []string) (map[string][]string, error) {
	out := map[string][]string{}
	if len(patterns) == 0 {
		patterns = []string{"./..."}
	}
	modules := map[string]string{}
	for _, d := range dirs {
		if raw, err := os.ReadFile(filepath.Join(tree, d, "go.mod")); err == nil {
			modules[d] = modulePath(string(raw))
		}
	}
	for _, p := range patterns {
		if strings.HasPrefix(p, "./") || p == "." {
			for _, d := range dirs {
				out[d] = append(out[d], p)
			}
			continue
		}
		owner, best := "", -1
		for _, d := range dirs {
			mp := modules[d]
			if mp != "" && (p == mp || strings.HasPrefix(p, mp+"/")) && len(mp) > best {
				owner, best = d, len(mp)
			}
		}
		if best < 0 {
			return nil, fmt.Errorf("no module of the workspace holds the package %s", p)
		}
		out[owner] = append(out[owner], p)
	}
	return out, nil
}

// modulePath is the module path a go.mod declares, "" when it declares none.
func modulePath(gomod string) string {
	for _, l := range strings.Split(gomod, "\n") {
		if f := strings.Fields(l); len(f) == 2 && f[0] == "module" {
			return strings.Trim(f[1], `"`)
		}
	}
	return ""
}

// runPatternOK says whether a -run pattern is one go test takes as a single argument: no flag, no space, no control
// character, and short ("" is every test).
func runPatternOK(p string) bool {
	if len(p) > 512 || strings.HasPrefix(p, "-") {
		return false
	}
	for _, r := range p {
		if r <= ' ' || r == 0x7f {
			return false
		}
	}
	if p != "" {
		if _, err := regexp.Compile(strings.Split(p, "/")[0]); err != nil {
			return false
		}
	}
	return true
}