140 lines
6.2 KiB
Go
140 lines
6.2 KiB
Go
package main
|
|
|
|
// The judging of a test level (novox/hq ADR 0445 §2 §3; issues 1290 and 1486): `go test -json` read into a result
|
|
// that says, per package, what ran and for how long, which tests failed and why, and whether the level kept its
|
|
// bound. A level that failed is still a finished step: the verdict is in the result, and a step that ends failed is a
|
|
// step that could not run.
|
|
|
|
import (
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
|
|
sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep"
|
|
)
|
|
|
|
// The two levels (ADR 0445 §2).
|
|
const (
|
|
LevelUnit = "unit"
|
|
LevelIntegration = "integration"
|
|
)
|
|
|
|
// UnitBound is the unit level's one bound: its running time, five seconds for the level, no setup (ADR 0445 §3). The
|
|
// level's time is that of its slowest package, as packages run side by side, the compile not counted.
|
|
const UnitBound = 5 * time.Second
|
|
|
|
// Bounds are the bounds a level is judged by: Setup is the time of everything before the tests run (compiling,
|
|
// starting the store and the bus; the unit level has none) and Run the tests' own time.
|
|
type Bounds struct {
|
|
Setup, Run time.Duration
|
|
}
|
|
|
|
// PackageResult, TestTime and Failure are the SDK's failure contract (buildstep, issue 1901), with the JSON names this
|
|
// module always sent: a Failure names its test, its file, its cause (never empty, never a passing line) and its own
|
|
// output up to 4096 bytes, cause first.
|
|
type (
|
|
PackageResult = buildstep.GoPackage
|
|
TestTime = buildstep.TestTime
|
|
Failure = sdkstep.Failure
|
|
)
|
|
|
|
// TestResult is what a test step passes with.
|
|
type TestResult struct {
|
|
Level string `json:"level"`
|
|
Passed bool `json:"passed"`
|
|
// Why says, in words, why the level failed, or that it has no tests.
|
|
Why string `json:"why,omitempty"`
|
|
NoTests bool `json:"no_tests,omitempty"`
|
|
Tests int `json:"tests"`
|
|
Skipped int `json:"skipped"`
|
|
Packages []PackageResult `json:"packages"`
|
|
// Elapsed is the level's running time in seconds: the slowest package's.
|
|
Elapsed float64 `json:"elapsed"`
|
|
// Setup is the level's setup time in seconds, when it has a setup.
|
|
Setup float64 `json:"setup,omitempty"`
|
|
// Lint is the unit level's gofmt and go vet time in seconds, before its tests (ADR 0529 §1).
|
|
Lint float64 `json:"lint,omitempty"`
|
|
// Bound and SetupBound are the bounds the level was judged by, in seconds.
|
|
Bound float64 `json:"bound"`
|
|
SetupBound float64 `json:"setup_bound,omitempty"`
|
|
Slowest []TestTime `json:"slowest,omitempty"`
|
|
Failed []Failure `json:"failed,omitempty"`
|
|
// Store and Bus are what the integration level ran against: the postgres major version and the nats-server
|
|
// version (ADR 0470 §4).
|
|
// OverBound says a bound was kept no longer, and OverWhich which: setup, run or both. The level's verdict, Passed, is
|
|
// the tests' own: the checker judges the bounds (ADR 0453 §9), with the slowest tests this result names.
|
|
OverBound bool `json:"over_bound"`
|
|
OverWhich string `json:"over_which,omitempty"`
|
|
Store string `json:"store,omitempty"`
|
|
Bus string `json:"bus,omitempty"`
|
|
// JudgedBy is the program that ran the level (ADR 0535 §5): "installed", or "candidate <merge commit>" under the
|
|
// runner workspace.
|
|
JudgedBy string `json:"judged_by,omitempty"`
|
|
// OutOfStep are the generated files of a contracts module in the workspace out of step with what the generator writes
|
|
// (CAT-BG5, buildstep.OutOfStep): the unit level fails on any, Why naming each. Absent when none is.
|
|
OutOfStep []string `json:"out-of-step,omitempty"`
|
|
// By is the build module and the SDK it was built with (sdkstep.By): the checker names it when a failure hides
|
|
// its cause.
|
|
By string `json:"by,omitempty"`
|
|
}
|
|
|
|
// readGoTest reads one go.mod directory's `go test -json` into res through the SDK's reader (issue 1901: one failure
|
|
// contract for every build module, sdkstep.Failure), and answers the lines that were no event and how many packages
|
|
// it named.
|
|
func readGoTest(res *TestResult, dir, out string) (noise []string, packages int) {
|
|
read := buildstep.ReadGoTest(out, dir)
|
|
res.Tests += read.Tests
|
|
res.Skipped += read.Skipped
|
|
res.Failed = append(res.Failed, read.Failed...)
|
|
res.Packages = append(res.Packages, read.Packages...)
|
|
res.Slowest = append(res.Slowest, read.Times...)
|
|
res.Elapsed = max(res.Elapsed, read.Elapsed)
|
|
return read.Noise, len(read.Packages)
|
|
}
|
|
|
|
// judge sets Passed and Why from what was read and the bounds. A level with no failure, over no bound, passes; with no
|
|
// test at all it says the repository has none at this level, which is neither a failure nor a warning (ADR 0445 §1).
|
|
func (res *TestResult) judge(b Bounds) {
|
|
sort.SliceStable(res.Slowest, func(i, j int) bool { return res.Slowest[i].Elapsed > res.Slowest[j].Elapsed })
|
|
if len(res.Slowest) > 5 {
|
|
res.Slowest = res.Slowest[:5]
|
|
}
|
|
res.Bound, res.SetupBound = b.Run.Seconds(), b.Setup.Seconds()
|
|
var why, over []string
|
|
if n := len(res.Failed); n > 0 {
|
|
names := []string{}
|
|
for _, f := range res.Failed {
|
|
names = append(names, strings.TrimSpace(f.Package+" "+f.Test))
|
|
}
|
|
why = append(why, fmt.Sprintf("%d failed: %s", n, strings.Join(names, ", ")))
|
|
}
|
|
res.Passed = len(res.Failed) == 0
|
|
if b.Setup > 0 && res.Setup > b.Setup.Seconds() {
|
|
over = append(over, "setup")
|
|
why = append(why, fmt.Sprintf("the %s level's setup took %.2f s, over its setup bound of %s", res.Level, res.Setup, seconds(b.Setup)))
|
|
}
|
|
if b.Run > 0 && res.Elapsed > b.Run.Seconds() {
|
|
over = append(over, "run")
|
|
slow := []string{}
|
|
for _, s := range res.Slowest {
|
|
slow = append(slow, fmt.Sprintf("%s %s (%.2f s)", s.Package, s.Test, s.Elapsed))
|
|
}
|
|
why = append(why, fmt.Sprintf("the %s level took %.2f s, over its bound of %s; the slowest tests: %s", res.Level, res.Elapsed, seconds(b.Run), strings.Join(slow, ", ")))
|
|
}
|
|
res.OverBound, res.OverWhich = len(over) > 0, strings.Join(over, ",")
|
|
res.NoTests = res.Tests == 0 && len(res.Failed) == 0
|
|
switch {
|
|
case len(why) > 0:
|
|
res.Why = strings.Join(why, "; ")
|
|
case res.NoTests:
|
|
res.Why = "the repository has no tests at the " + res.Level + " level"
|
|
}
|
|
}
|
|
|
|
// seconds is a duration as seconds, without trailing zeros: "5 s", "0.01 s".
|
|
func seconds(d time.Duration) string {
|
|
return strings.TrimRight(strings.TrimRight(fmt.Sprintf("%.3f", d.Seconds()), "0"), ".") + " s"
|
|
}
|