jschoubben
6fe7b7dd6c
The checker stopped reading a level's test results at the first that did not pass and kept only its first failure, so a check named one fault per round. It now reads and counts every result and writes every failed test with its first line into the verdict's Failures and the try's failed list (at most 50, then how many more); Failed keeps the first one's words whole (issue 1901). Pins mesh-sdk fix/1921-every-failure.
91 lines
4.3 KiB
Go
91 lines
4.3 KiB
Go
package check
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"strings"
|
|
"testing"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
)
|
|
|
|
// failingResult is a level's test result in which the named tests failed, each in its own package.
|
|
func failingResult(t *testing.T, tests int, failed ...Failure) json.RawMessage {
|
|
t.Helper()
|
|
raw, err := json.Marshal(Tested{Level: delivery.LevelIntegration, Tests: tests, Elapsed: 2, Failed: failed, Passed: len(failed) == 0})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return raw
|
|
}
|
|
|
|
func judgeLevel(t *testing.T, tests ...json.RawMessage) (*delivery.Verdict, *state) {
|
|
t.Helper()
|
|
st := &state{j: &Examiner{}, c: &Case{}, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{}}
|
|
v, _ := st.level(t.Context(), &target{pull: Pull{Repository: "novox/x"}}, delivery.LevelAnswer{Level: delivery.LevelIntegration, Tests: tests})
|
|
return v, st
|
|
}
|
|
|
|
// Issue 1921: two failing tests in two packages are two entries of the verdict's Failures and two lines of a try's
|
|
// failed list, each with its own first line (its cause, issue 1901), whether the build module answered them in one test
|
|
// result (one go test run over both packages) or in two (the changed packages first, then the rest); every result is
|
|
// counted, the ones after a failure too; and Failed keeps the first one's words whole.
|
|
func TestTwoFailingTestsInTwoPackagesAreTwoEntries(t *testing.T) {
|
|
a := Failure{Package: "x/a", Test: "TestA", Cause: "a_test.go:9: want 1, got 2", Output: " a_test.go:3: logged\n a_test.go:9: want 1, got 2\n--- FAIL: TestA (0.00s)"}
|
|
b := Failure{Package: "x/b", Test: "TestB", Cause: "b_test.go:4: boom", Output: " b_test.go:4: boom\n--- FAIL: TestB (0.00s)"}
|
|
for name, tests := range map[string][]json.RawMessage{
|
|
"one result": {failingResult(t, 10, a, b)},
|
|
"two results": {failingResult(t, 4, a), failingResult(t, 3), failingResult(t, 3, b)},
|
|
} {
|
|
v, st := judgeLevel(t, tests...)
|
|
if v == nil {
|
|
t.Fatalf("%s: the level passed", name)
|
|
}
|
|
want := []string{"integration: x/a.TestA: a_test.go:9: want 1, got 2", "integration: x/b.TestB: b_test.go:4: boom"}
|
|
if got := failedLines(*v); strings.Join(got, "\n") != strings.Join(want, "\n") {
|
|
t.Errorf("%s: failed lines\n%s\nwant\n%s", name, strings.Join(got, "\n"), strings.Join(want, "\n"))
|
|
}
|
|
if len(v.Failures) != 2 || v.Failed.Test != "integration: x/a.TestA" || !strings.Contains(v.Failed.Words, "a_test.go:3: logged") {
|
|
t.Errorf("%s: verdict %+v", name, v)
|
|
}
|
|
if run := st.levels[delivery.LevelIntegration].run; run != 10 {
|
|
t.Errorf("%s: %d tests counted, want all 10", name, run)
|
|
}
|
|
if _, err := delivery.EncodeVerdict(delivery.Verdict{Group: "g", Heads: "0123456789ab", State: v.State, Failed: v.Failed, Failures: v.Failures}); err != nil {
|
|
t.Errorf("%s: the verdict is not written: %v", name, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Issue 1921: past delivery.MaxFailures the list names the first fifty and how many more, the status line says how many
|
|
// failed, and each entry's line is cut to MaxFailureLine.
|
|
func TestFailuresAreCappedWithHowManyMore(t *testing.T) {
|
|
var fs []Failure
|
|
for i := range 60 {
|
|
fs = append(fs, Failure{Package: fmt.Sprintf("x/p%d", i), Test: "TestT", Cause: strings.Repeat("y", 500)})
|
|
}
|
|
v, _ := judgeLevel(t, failingResult(t, 60, fs...))
|
|
if len(v.Failures) != delivery.MaxFailures+1 || v.Failures[delivery.MaxFailures].Test != "and 10 more" {
|
|
t.Fatalf("%d entries, last %+v", len(v.Failures), v.Failures[len(v.Failures)-1])
|
|
}
|
|
if len(v.Failures[0].Words) > MaxFailureLine {
|
|
t.Errorf("an entry's line is %d bytes", len(v.Failures[0].Words))
|
|
}
|
|
if n := failedCount(*v); n != 60 {
|
|
t.Errorf("failedCount = %d, want 60", n)
|
|
}
|
|
cs := &Case{Pulls: []Pull{{Repository: "novox/x", Head: "0123456789abcdef", Trunk: "fedcba9876543210"}}}
|
|
if line := Line(*v, cs, cs.Pulls[0], nil, nil, nil); !strings.HasPrefix(line, "failed (60 tests): integration: x/p0.TestT") {
|
|
t.Errorf("line %q", line)
|
|
}
|
|
}
|
|
|
|
// A verdict that names no list (a conflict, a refused step, a verdict written before issue 1921) is one failed line, as
|
|
// before.
|
|
func TestAVerdictWithoutFailuresIsOneLine(t *testing.T) {
|
|
v := delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "g14", Words: "files in conflict: go.mod\nmore"}}
|
|
if got := failedLines(v); len(got) != 1 || got[0] != "g14: files in conflict: go.mod" {
|
|
t.Fatalf("%q", got)
|
|
}
|
|
}
|