Files
mesh-catalog/modules/mesh-build-go/cmd/mesh-build-go/levels.go
T

140 lines
6.2 KiB
Go

package main
// The judging of a test level (novox/hq ADR 0445 §2 §3; issues 1290 and 1486): `go test -json` read into a result
// that says, per package, what ran and for how long, which tests failed and why, and whether the level kept its
// bound. A level that failed is still a finished step: the verdict is in the result, and a step that ends failed is a
// step that could not run.
import (
"fmt"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep"
)
// The two levels (ADR 0445 §2).
const (
LevelUnit = "unit"
LevelIntegration = "integration"
)
// UnitBound is the unit level's one bound: its running time, five seconds for the level, no setup (ADR 0445 §3). The
// level's time is that of its slowest package, as packages run side by side, the compile not counted.
const UnitBound = 5 * time.Second
// Bounds are the bounds a level is judged by: Setup is the time of everything before the tests run (compiling,
// starting the store and the bus; the unit level has none) and Run the tests' own time.
type Bounds struct {
Setup, Run time.Duration
}
// PackageResult, TestTime and Failure are the SDK's failure contract (buildstep, issue 1901), with the JSON names this
// module always sent: a Failure names its test, its file, its cause (never empty, never a passing line) and its own
// output up to 4096 bytes, cause first.
type (
PackageResult = buildstep.GoPackage
TestTime = buildstep.TestTime
Failure = sdkstep.Failure
)
// TestResult is what a test step passes with.
type TestResult struct {
Level string `json:"level"`
Passed bool `json:"passed"`
// Why says, in words, why the level failed, or that it has no tests.
Why string `json:"why,omitempty"`
NoTests bool `json:"no_tests,omitempty"`
Tests int `json:"tests"`
Skipped int `json:"skipped"`
Packages []PackageResult `json:"packages"`
// Elapsed is the level's running time in seconds: the slowest package's.
Elapsed float64 `json:"elapsed"`
// Setup is the level's setup time in seconds, when it has a setup.
Setup float64 `json:"setup,omitempty"`
// Lint is the unit level's gofmt and go vet time in seconds, before its tests (ADR 0529 §1).
Lint float64 `json:"lint,omitempty"`
// Bound and SetupBound are the bounds the level was judged by, in seconds.
Bound float64 `json:"bound"`
SetupBound float64 `json:"setup_bound,omitempty"`
Slowest []TestTime `json:"slowest,omitempty"`
Failed []Failure `json:"failed,omitempty"`
// Store and Bus are what the integration level ran against: the postgres major version and the nats-server
// version (ADR 0470 §4).
// OverBound says a bound was kept no longer, and OverWhich which: setup, run or both. The level's verdict, Passed, is
// the tests' own: the checker judges the bounds (ADR 0453 §9), with the slowest tests this result names.
OverBound bool `json:"over_bound"`
OverWhich string `json:"over_which,omitempty"`
Store string `json:"store,omitempty"`
Bus string `json:"bus,omitempty"`
// JudgedBy is the program that ran the level (ADR 0535 §5): "installed", or "candidate <merge commit>" under the
// runner workspace.
JudgedBy string `json:"judged_by,omitempty"`
// OutOfStep are the generated files of a contracts module in the workspace out of step with what the generator writes
// (CAT-BG5, buildstep.OutOfStep): the unit level fails on any, Why naming each. Absent when none is.
OutOfStep []string `json:"out-of-step,omitempty"`
// By is the build module and the SDK it was built with (sdkstep.By): the checker names it when a failure hides
// its cause.
By string `json:"by,omitempty"`
}
// readGoTest reads one go.mod directory's `go test -json` into res through the SDK's reader (issue 1901: one failure
// contract for every build module, sdkstep.Failure), and answers the lines that were no event and how many packages
// it named.
func readGoTest(res *TestResult, dir, out string) (noise []string, packages int) {
read := buildstep.ReadGoTest(out, dir)
res.Tests += read.Tests
res.Skipped += read.Skipped
res.Failed = append(res.Failed, read.Failed...)
res.Packages = append(res.Packages, read.Packages...)
res.Slowest = append(res.Slowest, read.Times...)
res.Elapsed = max(res.Elapsed, read.Elapsed)
return read.Noise, len(read.Packages)
}
// judge sets Passed and Why from what was read and the bounds. A level with no failure, over no bound, passes; with no
// test at all it says the repository has none at this level, which is neither a failure nor a warning (ADR 0445 §1).
func (res *TestResult) judge(b Bounds) {
sort.SliceStable(res.Slowest, func(i, j int) bool { return res.Slowest[i].Elapsed > res.Slowest[j].Elapsed })
if len(res.Slowest) > 5 {
res.Slowest = res.Slowest[:5]
}
res.Bound, res.SetupBound = b.Run.Seconds(), b.Setup.Seconds()
var why, over []string
if n := len(res.Failed); n > 0 {
names := []string{}
for _, f := range res.Failed {
names = append(names, strings.TrimSpace(f.Package+" "+f.Test))
}
why = append(why, fmt.Sprintf("%d failed: %s", n, strings.Join(names, ", ")))
}
res.Passed = len(res.Failed) == 0
if b.Setup > 0 && res.Setup > b.Setup.Seconds() {
over = append(over, "setup")
why = append(why, fmt.Sprintf("the %s level's setup took %.2f s, over its setup bound of %s", res.Level, res.Setup, seconds(b.Setup)))
}
if b.Run > 0 && res.Elapsed > b.Run.Seconds() {
over = append(over, "run")
slow := []string{}
for _, s := range res.Slowest {
slow = append(slow, fmt.Sprintf("%s %s (%.2f s)", s.Package, s.Test, s.Elapsed))
}
why = append(why, fmt.Sprintf("the %s level took %.2f s, over its bound of %s; the slowest tests: %s", res.Level, res.Elapsed, seconds(b.Run), strings.Join(slow, ", ")))
}
res.OverBound, res.OverWhich = len(over) > 0, strings.Join(over, ",")
res.NoTests = res.Tests == 0 && len(res.Failed) == 0
switch {
case len(why) > 0:
res.Why = strings.Join(why, "; ")
case res.NoTests:
res.Why = "the repository has no tests at the " + res.Level + " level"
}
}
// seconds is a duration as seconds, without trailing zeros: "5 s", "0.01 s".
func seconds(d time.Duration) string {
return strings.TrimRight(strings.TrimRight(fmt.Sprintf("%.3f", d.Seconds()), "0"), ".") + " s"
}