149 lines
6.8 KiB
Go
149 lines
6.8 KiB
Go
package conformance
|
|
|
|
import (
|
|
"encoding/json"
|
|
"strings"
|
|
"testing"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
|
)
|
|
|
|
// The failure contract (issue 1901): the SDK's own parsers keep it on every captured case of the kit (1901/AC6); the
|
|
// kit refuses what the build modules sent before it; the contract's own rules; and the Go reader's older guarantees.
|
|
func TestTheFailureContract(t *testing.T) {
|
|
t.Run("the SDK parsers keep it", func(t *testing.T) {
|
|
Run(t, GoTest, func(o Output) []buildstep.Failure { return buildstep.ReadGoTest(o.Stdout, ".").Failed })
|
|
Run(t, NodeTest, func(o Output) []buildstep.Failure { return buildstep.ReadTAP(".", "test", o.Dir, o.Report, o.Stdout) })
|
|
Run(t, TypeCheck, func(o Output) []buildstep.Failure {
|
|
return []buildstep.Failure{buildstep.ConsoleFailure(".", "type-check", o.Stdout+o.Stderr)}
|
|
})
|
|
})
|
|
|
|
// node's output cut to its first fifteen lines with no cause, whose first line was a passing test of the first file
|
|
// (mesh-build-node's lines() before the contract), and a Go failure keeping its first line.
|
|
t.Run("the kit refuses a failure that hides its cause", func(t *testing.T) {
|
|
firstLines := func(s string) string {
|
|
var keep []string
|
|
for _, l := range strings.Split(s, "\n") {
|
|
if l = strings.TrimSpace(l); l != "" && len(keep) < 15 {
|
|
keep = append(keep, l)
|
|
}
|
|
}
|
|
return strings.Join(keep, "\n")
|
|
}
|
|
for _, c := range []struct {
|
|
runner, name string
|
|
old func(Output) []buildstep.Failure
|
|
}{
|
|
{NodeTest, "two-files", func(o Output) []buildstep.Failure {
|
|
return []buildstep.Failure{{Package: ".", Test: "test", Output: firstLines(o.Report)}}
|
|
}},
|
|
{GoTest, "passfirst", func(o Output) []buildstep.Failure {
|
|
return []buildstep.Failure{{Package: "example.com/passfirst/b", Test: "TestTheSecondPackageFails", Cause: "ok \texample.com/passfirst/a", Output: "--- PASS: TestTheSecondPackagePassesFirst"}}
|
|
}},
|
|
} {
|
|
for _, k := range Cases(c.runner) {
|
|
if k.Name != c.name {
|
|
continue
|
|
}
|
|
out, err := Load(k)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if problems := Judge(k, c.old(out)); len(problems) == 0 {
|
|
t.Errorf("%s %s: the kit passed a failure that hides its cause", c.runner, c.name)
|
|
}
|
|
}
|
|
}
|
|
})
|
|
|
|
// What a passing line is (a test's own line starting "ok " is none), when a failure hides its cause, and that a long
|
|
// failure keeps its last lines within the bound and starts with a real line for an older checker, which reads only
|
|
// the first line of the output cut to 120 characters (1901/AC1, AC3).
|
|
t.Run("its rules", func(t *testing.T) {
|
|
for _, c := range []struct {
|
|
line string
|
|
passing bool
|
|
}{
|
|
{"✔ the same merge has the same id (8.0ms)", true}, {"ok 1 - passes", true}, {" ok 3 - inside", true},
|
|
{"ok \texample.com/a\t0.01s", true}, {"PASS", true}, {"--- PASS: TestA (0.00s)", true},
|
|
{" --- PASS: TestA/sub (0.00s)", true}, {"✖ fails (1ms)", false}, {"not ok 2 - fails", false},
|
|
{"okay, the seat moved", false}, {"ok then: the store refused the write", false}, {"ok the seat", false}, {"FAIL", false},
|
|
} {
|
|
if buildstep.PassingLine(c.line) != c.passing {
|
|
t.Errorf("PassingLine(%q) = %v, want %v", c.line, !c.passing, c.passing)
|
|
}
|
|
}
|
|
if buildstep.Hidden(buildstep.Failure{Cause: "the seat has 2 holders"}) != "" || buildstep.Hidden(buildstep.Failure{}) == "" ||
|
|
buildstep.Hidden(buildstep.Failure{Cause: "✔ passes (1ms)"}) == "" {
|
|
t.Error("Hidden does not tell a real cause from none and from a passing line")
|
|
}
|
|
var lines []string
|
|
for i := 0; i < 400; i++ {
|
|
lines = append(lines, " x_test.go:1: log line "+strings.Repeat("y", 40))
|
|
}
|
|
lines = append(lines, " x_test.go:9: the real cause", "--- FAIL: TestX (0.00s)")
|
|
oldFirst := func(out string) string { // the older checker's rule (judge.go firstLine before issue 1901)
|
|
for _, l := range strings.Split(out, "\n") {
|
|
if l = strings.TrimSpace(l); l != "" {
|
|
return l[:min(len(l), 120)]
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
for _, cause := range []string{"the real cause", ""} {
|
|
out := buildstep.Clip(lines, cause)
|
|
if len(out) > buildstep.OutputBytes+buildstep.LineBytes {
|
|
t.Errorf("1901/AC1: kept %d bytes, bound is %d", len(out), buildstep.OutputBytes)
|
|
}
|
|
if !strings.HasSuffix(out, "the real cause\n--- FAIL: TestX (0.00s)") || !strings.Contains(out, "\n... (") {
|
|
t.Errorf("1901/AC1: the last lines or the marker are lost: %q...%q", out[:40], out[len(out)-60:])
|
|
}
|
|
if got := oldFirst(out); strings.HasPrefix(got, "... (") || got == "" || (cause != "" && got != cause) {
|
|
t.Errorf("1901/AC3: cause %q: an older checker would read %q", cause, got)
|
|
}
|
|
}
|
|
})
|
|
|
|
// The Go reader (moved from mesh-build-go): an older checker, which knows no "cause" field, still reads a failure and
|
|
// gets at least the lines it kept (1901/AC3); a test's indented log line saying "panic:" is not the cause, go's own at
|
|
// column 0 is (1901/AC4); and each package counts its own skipped tests (issue 890).
|
|
t.Run("the Go reader", func(t *testing.T) {
|
|
read := func(name string) string {
|
|
raw, err := captured.ReadFile("captured/go/" + name + ".jsonl")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return string(raw)
|
|
}
|
|
b, _ := json.Marshal(buildstep.ReadGoTest(read("logthenfatal"), ".").Failed)
|
|
var old []struct {
|
|
Output string `json:"output,omitempty"`
|
|
}
|
|
if err := json.Unmarshal(b, &old); err != nil || len(old) != 1 || !strings.Contains(old[0].Output, "judged against") || !strings.Contains(old[0].Output, "3 subscription(s)") {
|
|
t.Errorf("1901/AC3: an older checker reads %+v, %v", old, err)
|
|
}
|
|
lines := strings.SplitAfter(read("deeptimeout"), "\n")
|
|
added := `{"Action":"output","Package":"example.com/deeptimeout","Test":"TestHoldsTheSeat","Output":" p_test.go:9: the child said:\n"}` + "\n" +
|
|
`{"Action":"output","Package":"example.com/deeptimeout","Test":"TestHoldsTheSeat","Output":" panic: child retried and recovered\n"}` + "\n" +
|
|
`{"Action":"output","Package":"example.com/deeptimeout","Test":"TestHoldsTheSeat","Output":" WARNING: DATA RACE\n"}` + "\n"
|
|
if got := buildstep.ReadGoTest(strings.Join(lines[:10], "")+added+strings.Join(lines[10:], ""), ".").Failed; len(got) != 1 || !strings.HasPrefix(got[0].Cause, "panic: test timed out after 2s") {
|
|
t.Errorf("1901/AC4: failures = %+v, want the timeout's panic as the cause", got)
|
|
}
|
|
res := buildstep.ReadGoTest(`{"Action":"run","Package":"m/a","Test":"TestA"}
|
|
{"Action":"skip","Package":"m/a","Test":"TestA","Elapsed":0}
|
|
{"Action":"run","Package":"m/b","Test":"TestB"}
|
|
{"Action":"pass","Package":"m/b","Test":"TestB","Elapsed":0}
|
|
{"Action":"pass","Package":"m/a","Elapsed":0}
|
|
{"Action":"pass","Package":"m/b","Elapsed":0}
|
|
`, ".")
|
|
got := map[string]int{}
|
|
for _, p := range res.Packages {
|
|
got[p.Package] = p.Skipped
|
|
}
|
|
if res.Skipped != 1 || got["m/a"] != 1 || got["m/b"] != 0 {
|
|
t.Errorf("skipped = %d, per package %v, want 1 with m/a 1 and m/b 0", res.Skipped, got)
|
|
}
|
|
})
|
|
}
|