mesh-sdk/go/buildstep/conformance/conformance_test.go

package conformance

import (
	"context"
	"encoding/json"
	"strings"
	"testing"
	"time"

	"git.novox.be/novox/mesh-sdk/go/buildstep"
)

// The failure contract (issue 1901): the SDK's own parsers keep it on every captured case of the kit (1901/AC6); the
// kit refuses what the build modules sent before it; the contract's own rules; and the Go reader's older guarantees.
func TestTheFailureContract(t *testing.T) {
	t.Run("the SDK parsers keep it", func(t *testing.T) {
		Run(t, GoTest, func(o Output) []buildstep.Failure { return buildstep.ReadGoTest(o.Stdout, ".").Failed })
		Run(t, NodeTest, func(o Output) []buildstep.Failure { return buildstep.ReadTAP(".", "test", o.Dir, o.Report, o.Stdout) })
		Run(t, TypeCheck, func(o Output) []buildstep.Failure {
			return []buildstep.Failure{buildstep.ConsoleFailure(".", "type-check", o.Stdout+o.Stderr)}
		})
	})

	// 1890/AC7: the SDK's start sweep keeps the left-running case, and the kit refuses a sweep that ends nothing or ends
	// everything.
	t.Run("the left-running case", func(t *testing.T) {
		LeftRunning(t, func(ctx context.Context, scan buildstep.Scan, node string, before time.Time, logf func(string, ...any)) {
			buildstep.EndStaleAtStart(ctx, scan, node, before, logf)
		})
		if p := JudgeLeftRunning(func(context.Context, buildstep.Scan, string, time.Time, func(string, ...any)) {}); len(p) != 11 {
			t.Errorf("a sweep that ends nothing: %d problems, want 11 (three steps not ended and unsaid, one owner unnamed, two asks not deleted and unsaid):\n%s", len(p), strings.Join(p, "\n"))
		}
		wrongNode := func(ctx context.Context, scan buildstep.Scan, _ string, before time.Time, logf func(string, ...any)) {
			buildstep.EndStaleAtStart(ctx, scan, "kitother", before, logf)
		}
		if p := JudgeLeftRunning(wrongNode); len(p) == 0 {
			t.Error("a sweep of another node's steps is not refused")
		}
	})

	// node's output cut to its first fifteen lines with no cause, whose first line was a passing test of the first file
	// (mesh-build-node's lines() before the contract), and a Go failure keeping its first line.
	t.Run("the kit refuses a failure that hides its cause", func(t *testing.T) {
		firstLines := func(s string) string {
			var keep []string
			for _, l := range strings.Split(s, "\n") {
				if l = strings.TrimSpace(l); l != "" && len(keep) < 15 {
					keep = append(keep, l)
				}
			}
			return strings.Join(keep, "\n")
		}
		for _, c := range []struct {
			runner, name string
			old          func(Output) []buildstep.Failure
		}{
			{NodeTest, "two-files", func(o Output) []buildstep.Failure {
				return []buildstep.Failure{{Package: ".", Test: "test", Output: firstLines(o.Report)}}
			}},
			{GoTest, "passfirst", func(o Output) []buildstep.Failure {
				return []buildstep.Failure{{Package: "example.com/passfirst/b", Test: "TestTheSecondPackageFails", Cause: "ok  \texample.com/passfirst/a", Output: "--- PASS: TestTheSecondPackagePassesFirst"}}
			}},
		} {
			for _, k := range Cases(c.runner) {
				if k.Name != c.name {
					continue
				}
				out, err := Load(k)
				if err != nil {
					t.Fatal(err)
				}
				if problems := Judge(k, c.old(out)); len(problems) == 0 {
					t.Errorf("%s %s: the kit passed a failure that hides its cause", c.runner, c.name)
				}
			}
		}
	})

	// What a passing line is (a test's own line starting "ok " is none), when a failure hides its cause, and that a long
	// failure keeps its last lines within the bound and starts with a real line for an older checker, which reads only
	// the first line of the output cut to 120 characters (1901/AC1, AC3).
	t.Run("its rules", func(t *testing.T) {
		for _, c := range []struct {
			line    string
			passing bool
		}{
			{"✔ the same merge has the same id (8.0ms)", true}, {"ok 1 - passes", true}, {"    ok 3 - inside", true},
			{"ok  \texample.com/a\t0.01s", true}, {"PASS", true}, {"--- PASS: TestA (0.00s)", true},
			{"    --- PASS: TestA/sub (0.00s)", true}, {"✖ fails (1ms)", false}, {"not ok 2 - fails", false},
			{"okay, the seat moved", false}, {"ok then: the store refused the write", false}, {"ok  the seat", false}, {"FAIL", false},
		} {
			if buildstep.PassingLine(c.line) != c.passing {
				t.Errorf("PassingLine(%q) = %v, want %v", c.line, !c.passing, c.passing)
			}
		}
		if buildstep.Hidden(buildstep.Failure{Cause: "the seat has 2 holders"}) != "" || buildstep.Hidden(buildstep.Failure{}) == "" ||
			buildstep.Hidden(buildstep.Failure{Cause: "✔ passes (1ms)"}) == "" {
			t.Error("Hidden does not tell a real cause from none and from a passing line")
		}
		var lines []string
		for i := 0; i < 400; i++ {
			lines = append(lines, "    x_test.go:1: log line "+strings.Repeat("y", 40))
		}
		lines = append(lines, "    x_test.go:9: the real cause", "--- FAIL: TestX (0.00s)")
		oldFirst := func(out string) string { // the older checker's rule (judge.go firstLine before issue 1901)
			for _, l := range strings.Split(out, "\n") {
				if l = strings.TrimSpace(l); l != "" {
					return l[:min(len(l), 120)]
				}
			}
			return ""
		}
		for _, cause := range []string{"the real cause", ""} {
			out := buildstep.Clip(lines, cause)
			if len(out) > buildstep.OutputBytes+buildstep.LineBytes {
				t.Errorf("1901/AC1: kept %d bytes, bound is %d", len(out), buildstep.OutputBytes)
			}
			if !strings.HasSuffix(out, "the real cause\n--- FAIL: TestX (0.00s)") || !strings.Contains(out, "\n... (") {
				t.Errorf("1901/AC1: the last lines or the marker are lost: %q...%q", out[:40], out[len(out)-60:])
			}
			if got := oldFirst(out); strings.HasPrefix(got, "... (") || got == "" || (cause != "" && got != cause) {
				t.Errorf("1901/AC3: cause %q: an older checker would read %q", cause, got)
			}
		}
	})

	// The Go reader (moved from mesh-build-go): an older checker, which knows no "cause" field, still reads a failure and
	// gets at least the lines it kept (1901/AC3); a test's indented log line saying "panic:" is not the cause, go's own at
	// column 0 is (1901/AC4); and each package counts its own skipped tests (issue 890).
	t.Run("the Go reader", func(t *testing.T) {
		read := func(name string) string {
			raw, err := captured.ReadFile("captured/go/" + name + ".jsonl")
			if err != nil {
				t.Fatal(err)
			}
			return string(raw)
		}
		b, _ := json.Marshal(buildstep.ReadGoTest(read("logthenfatal"), ".").Failed)
		var old []struct {
			Output string `json:"output,omitempty"`
		}
		if err := json.Unmarshal(b, &old); err != nil || len(old) != 1 || !strings.Contains(old[0].Output, "judged against") || !strings.Contains(old[0].Output, "3 subscription(s)") {
			t.Errorf("1901/AC3: an older checker reads %+v, %v", old, err)
		}
		lines := strings.SplitAfter(read("deeptimeout"), "\n")
		added := `{"Action":"output","Package":"example.com/deeptimeout","Test":"TestHoldsTheSeat","Output":"    p_test.go:9: the child said:\n"}` + "\n" +
			`{"Action":"output","Package":"example.com/deeptimeout","Test":"TestHoldsTheSeat","Output":"        panic: child retried and recovered\n"}` + "\n" +
			`{"Action":"output","Package":"example.com/deeptimeout","Test":"TestHoldsTheSeat","Output":"        WARNING: DATA RACE\n"}` + "\n"
		if got := buildstep.ReadGoTest(strings.Join(lines[:10], "")+added+strings.Join(lines[10:], ""), ".").Failed; len(got) != 1 || !strings.HasPrefix(got[0].Cause, "panic: test timed out after 2s") {
			t.Errorf("1901/AC4: failures = %+v, want the timeout's panic as the cause", got)
		}
		res := buildstep.ReadGoTest(`{"Action":"run","Package":"m/a","Test":"TestA"}
{"Action":"skip","Package":"m/a","Test":"TestA","Elapsed":0}
{"Action":"run","Package":"m/b","Test":"TestB"}
{"Action":"pass","Package":"m/b","Test":"TestB","Elapsed":0}
{"Action":"pass","Package":"m/a","Elapsed":0}
{"Action":"pass","Package":"m/b","Elapsed":0}
`, ".")
		got := map[string]int{}
		for _, p := range res.Packages {
			got[p.Package] = p.Skipped
		}
		if res.Skipped != 1 || got["m/a"] != 1 || got["m/b"] != 0 {
			t.Errorf("skipped = %d, per package %v, want 1 with m/a 1 and m/b 0", res.Skipped, got)
		}
	})
}