jschoubben
e40dd0f56a
Each build module parsed its own test runner's output, so a fix in one never reached the others: a failed node --test run was worded by a passing line of its first file. buildstep.Failure names the test, its file, its cause and its own output; ReadGoTest (moved from mesh-build-go) and ReadTAP fill it; package conformance holds every build module to it with runner output captured with the build nodes' own toolchains.
99 lines
4.4 KiB
Go
99 lines
4.4 KiB
Go
package buildstep
|
|
|
|
import (
|
|
"encoding/json"
|
|
"os"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// The contract's own rules (issue 1901): what a passing line is, when a failure hides its cause, and that a long failure
|
|
// keeps its last lines within the bound and starts with a real line for an older checker, which reads only the first
|
|
// line of the output cut to 120 characters (1901/AC1, AC3, moved from mesh-build-go).
|
|
func TestTheFailureContractsRules(t *testing.T) {
|
|
for _, c := range []struct {
|
|
line string
|
|
passing bool
|
|
}{
|
|
{"✔ the same merge has the same id (8.0ms)", true}, {"ok 1 - passes", true}, {"ok \texample.com/a\t0.01s", true},
|
|
{"PASS", true}, {"--- PASS: TestA (0.00s)", true}, {" --- PASS: TestA/sub (0.00s)", true},
|
|
{"✖ fails (1ms)", false}, {"not ok 2 - fails", false}, {"okay, the seat moved", false}, {"FAIL", false},
|
|
} {
|
|
if PassingLine(c.line) != c.passing {
|
|
t.Errorf("PassingLine(%q) = %v, want %v", c.line, !c.passing, c.passing)
|
|
}
|
|
}
|
|
if Hidden(Failure{Cause: "the seat has 2 holders"}) != "" || Hidden(Failure{}) == "" || Hidden(Failure{Cause: "✔ passes (1ms)"}) == "" {
|
|
t.Error("Hidden does not tell a real cause from none and from a passing line")
|
|
}
|
|
|
|
var lines []string
|
|
for i := 0; i < 400; i++ {
|
|
lines = append(lines, " x_test.go:1: log line "+strings.Repeat("y", 40))
|
|
}
|
|
lines = append(lines, " x_test.go:9: the real cause", "--- FAIL: TestX (0.00s)")
|
|
oldFirst := func(out string) string { // the older checker's rule (judge.go firstLine before issue 1901)
|
|
for _, l := range strings.Split(out, "\n") {
|
|
if l = strings.TrimSpace(l); l != "" {
|
|
return l[:min(len(l), 120)]
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
for _, cause := range []string{"the real cause", ""} {
|
|
out := Clip(lines, cause)
|
|
if len(out) > OutputBytes+LineBytes {
|
|
t.Errorf("1901/AC1: kept %d bytes, bound is %d", len(out), OutputBytes)
|
|
}
|
|
if !strings.HasSuffix(out, "the real cause\n--- FAIL: TestX (0.00s)") || !strings.Contains(out, "\n... (") {
|
|
t.Errorf("1901/AC1: the last lines or the marker are lost: %q...%q", out[:40], out[len(out)-60:])
|
|
}
|
|
if got := oldFirst(out); strings.HasPrefix(got, "... (") || got == "" || (cause != "" && got != cause) {
|
|
t.Errorf("1901/AC3: cause %q: an older checker would read %q", cause, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
// The Go reader (moved from mesh-build-go): an older checker, which knows no "cause" field, still reads a failure and
|
|
// gets at least the lines it kept (1901/AC3); a test's indented log line saying "panic:" is not the cause, go's own at
|
|
// column 0 is (1901/AC4); and each package counts its own skipped tests (issue 890).
|
|
func TestTheGoReader(t *testing.T) {
|
|
read := func(name string) string {
|
|
raw, err := os.ReadFile("conformance/captured/go/" + name + ".jsonl")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return string(raw)
|
|
}
|
|
b, _ := json.Marshal(ReadGoTest(read("logthenfatal"), ".").Failed)
|
|
var old []struct {
|
|
Output string `json:"output,omitempty"`
|
|
}
|
|
if err := json.Unmarshal(b, &old); err != nil || len(old) != 1 || !strings.Contains(old[0].Output, "judged against") || !strings.Contains(old[0].Output, "3 subscription(s)") {
|
|
t.Errorf("1901/AC3: an older checker reads %+v, %v", old, err)
|
|
}
|
|
|
|
lines := strings.SplitAfter(read("deeptimeout"), "\n")
|
|
added := `{"Action":"output","Package":"example.com/deeptimeout","Test":"TestHoldsTheSeat","Output":" p_test.go:9: the child said:\n"}` + "\n" +
|
|
`{"Action":"output","Package":"example.com/deeptimeout","Test":"TestHoldsTheSeat","Output":" panic: child retried and recovered\n"}` + "\n" +
|
|
`{"Action":"output","Package":"example.com/deeptimeout","Test":"TestHoldsTheSeat","Output":" WARNING: DATA RACE\n"}` + "\n"
|
|
if got := ReadGoTest(strings.Join(lines[:10], "")+added+strings.Join(lines[10:], ""), ".").Failed; len(got) != 1 || !strings.HasPrefix(got[0].Cause, "panic: test timed out after 2s") {
|
|
t.Errorf("1901/AC4: failures = %+v, want the timeout's panic as the cause", got)
|
|
}
|
|
|
|
res := ReadGoTest(`{"Action":"run","Package":"m/a","Test":"TestA"}
|
|
{"Action":"skip","Package":"m/a","Test":"TestA","Elapsed":0}
|
|
{"Action":"run","Package":"m/b","Test":"TestB"}
|
|
{"Action":"pass","Package":"m/b","Test":"TestB","Elapsed":0}
|
|
{"Action":"pass","Package":"m/a","Elapsed":0}
|
|
{"Action":"pass","Package":"m/b","Elapsed":0}
|
|
`, ".")
|
|
got := map[string]int{}
|
|
for _, p := range res.Packages {
|
|
got[p.Package] = p.Skipped
|
|
}
|
|
if res.Skipped != 1 || got["m/a"] != 1 || got["m/b"] != 0 {
|
|
t.Errorf("skipped = %d, per package %v, want 1 with m/a 1 and m/b 0", res.Skipped, got)
|
|
}
|
|
}
|