mesh-sdk/go/buildstep/conformance/conformance.go

// Package conformance is the kit every build module's tests run (novox/hq issue 1901, its thorough fix): real output of
// the test runners, captured with the build nodes' own toolchains (captured/README.md), and what a build module must say
// of each failure in it under the failure contract (buildstep.Failure). A build module's test calls Run with its own parse
// path, the code that turns a runner's output into the failures of its test result; the kit fails the test when a
// failure is not named, its cause is missing or wrong, or a passing line stands in its place.
//
// A catalogue check (mesh-catalog checks/failurekit) refuses a build module with a test step whose tests do not call Run,
// and any build module whose tests do not call LeftRunning, the kit's case of steps a previous process left running
// (leftrunning.go, issue 1890).
package conformance

import (
	"embed"
	"fmt"
	"strings"
	"testing"

	"git.novox.be/novox/mesh-sdk/go/buildstep"
)

// The runners the kit holds output of.
const (
	// GoTest is `go test -json -count=1 ./...`: Output.Stdout is its output (go1.27.3).
	GoTest = "go test -json"
	// NodeTest is a package's `npm test` running `node --test --experimental-strip-types` with
	// NODE_OPTIONS=buildstep.NodeTestOptions(<file>): Output.Report is the TAP file, Output.Stdout the script's own output
	// and stderr, Output.Dir the package's directory as it was captured (node v26.12.1, npm 12.2.1).
	NodeTest = "node --test"
	// TypeCheck is tsc's type-check (`tsc --noEmit`, typescript 5.9.3): Output.Stdout is what it wrote; its failure
	// names the test "type-check".
	TypeCheck = "tsc"
)

// Output is one captured run of a runner, as a build module's parse path takes it.
type Output struct {
	Stdout, Stderr string
	// Report is what a runner wrote to a report file of its own (node --test's TAP), "" for the others.
	Report string
	// Dir is the directory the run was captured in: a node test's file is said relative to it.
	Dir string
}

// Parse is a build module's own parse path: the failures its test result would carry for one run that failed.
type Parse func(Output) []buildstep.Failure

// Case is one captured run and what its failure must say.
type Case struct {
	Name   string
	Runner string
	// Test is the failing test the failure names (for a package that stopped, the tests that were running; "" for none).
	Test string
	// File, when set, is where the failing test is.
	File string
	// Cause holds what the failure's cause must contain.
	Cause []string
	// Passing are the names of tests that passed in the run, and what they logged: no failure's words may hold them.
	Passing []string
}

//go:embed captured
var captured embed.FS

// Cases are the kit's cases of a runner.
func Cases(runner string) []Case {
	var out []Case
	for _, c := range cases {
		if c.Runner == runner {
			out = append(out, c)
		}
	}
	return out
}

var cases = []Case{
	{Name: "passfirst", Runner: GoTest, Test: "TestTheSecondPackageFails", Cause: []string{"the seat has 2 holders, want 1"},
		Passing: []string{"TestTheFirstPackagePasses", "TestTheSecondPackagePassesFirst"}},
	{Name: "subtests", Runner: GoTest, Test: "TestTheTable/fails", Cause: []string{"got 3 machines, want 4"}, Passing: []string{"TestTheTable/passes"}},
	{Name: "logthenfatal", Runner: GoTest, Test: "TestThree", Cause: []string{"3 subscription(s) derive to a subject no emitter owns:"}},
	{Name: "panicking", Runner: GoTest, Test: "TestPanics", Cause: []string{"panic: assignment to entry in nil map"}},
	{Name: "timeout", Runner: GoTest, Test: "TestWaitsForever", Cause: []string{"panic: test timed out after 2s", "TestWaitsForever"}},
	{Name: "deeptimeout", Runner: GoTest, Test: "TestHoldsTheSeat, TestWaitsForTheBus", Cause: []string{"panic: test timed out after 2s", "TestHoldsTheSeat", "TestWaitsForTheBus"}},
	{Name: "buildfail", Runner: GoTest, Cause: []string{`cannot use "seat" (untyped string constant) as int value`}},
	{Name: "exiting", Runner: GoTest, Test: "TestExits", Cause: []string{"exited", "TestExits"}},
	{Name: "killed", Runner: GoTest, Test: "TestKilled", Cause: []string{"signal: killed"}},
	{Name: "passthenfail", Runner: GoTest, Cause: []string{"panic: the bus closed after the tests: nats: connection closed"}},
	{Name: "mainexit", Runner: GoTest, Cause: []string{"passed", "exited"}},
	{Name: "race", Runner: GoTest, Test: "TestRacesOnTheSeat", Cause: []string{"DATA RACE", "Write at", "example.com/race.TestRacesOnTheSeat.func1()", "p_test.go:13", "created at example.com/race.TestRacesOnTheSeat()"}},
	{Name: "subpanic", Runner: GoTest, Test: "TestTable/panics", Cause: []string{"panic: assignment to entry in nil map"}, Passing: []string{"TestTable/passes"}},
	{Name: "racemain", Runner: GoTest, Cause: []string{"DATA RACE", "Write at", "example.com/racemain.TestMain.func1()", "p_test.go:15", "created at example.com/racemain.TestMain()"}},

	{Name: "two-files", Runner: NodeTest, Test: "fails in the second file", File: "test/b.test.ts:4:1", Cause: []string{"one plus one is not three"},
		Passing: []string{"passes in the first file", "passes in the second file first"}},
	{Name: "suite", Runner: NodeTest, Test: "fails inside", File: "test/a.test.ts:5:3", Cause: []string{"Expected values to be strictly deep-equal:"},
		Passing: []string{"passes inside"}},
	{Name: "crash", Runner: NodeTest, Test: "test/b.test.ts", File: "test/b.test.ts:1:1", Cause: []string{"the bus closed under the test"},
		Passing: []string{"passes before the crash"}},
	{Name: "logcrash", Runner: NodeTest, Test: "test/b.test.ts", File: "test/b.test.ts:1:1", Cause: []string{"the bus closed under the test"},
		Passing: []string{"passes and logs", "retrying after Error: the store was not ready yet"}},
	{Name: "timeout", Runner: NodeTest, Test: "waits for an answer that never comes", File: "test/a.test.ts:3:1", Cause: []string{"test timed out after 200ms"},
		Passing: []string{"passes before the wait"}},
	{Name: "syntax", Runner: NodeTest, Test: "test/b.test.ts", File: "test/b.test.ts:1:1", Cause: []string{"SyntaxError", "Expected ',', got '<eof>'"},
		Passing: []string{"passes in the first file"}},

	{Name: "typeerror", Runner: TypeCheck, Test: "type-check", Cause: []string{"seat.ts(2,14): error TS2322: Type 'number' is not assignable to type 'string'."}},
}

// Load is the captured output of a case.
func Load(c Case) (Output, error) {
	switch c.Runner {
	case GoTest:
		b, err := captured.ReadFile("captured/go/" + c.Name + ".jsonl")
		return Output{Stdout: string(b)}, err
	case NodeTest:
		report, err := captured.ReadFile("captured/node/" + c.Name + ".tap")
		if err != nil {
			return Output{}, err
		}
		console, err := captured.ReadFile("captured/node/" + c.Name + ".console")
		return Output{Report: string(report), Stdout: string(console), Dir: "/tmp/1901k/node/" + c.Name}, err
	default:
		b, err := captured.ReadFile("captured/tsc/" + c.Name + ".console")
		return Output{Stdout: string(b), Dir: "/tmp/1901k/tsc/" + c.Name}, err
	}
}

// Run runs every case of runner through parse, each as a subtest, and fails a case whose failure does not keep the
// contract: exactly one failure, naming the failing test (and its file, where the runner says it), with a cause that
// holds what the case says and is no passing line, an output that starts with the cause, stays within its bound and
// holds no passing line and no passing test's name.
func Run(t *testing.T, runner string, parse Parse) {
	t.Helper()
	cs := Cases(runner)
	if len(cs) == 0 {
		t.Fatalf("the kit holds no case of the runner %q", runner)
	}
	for _, c := range cs {
		t.Run(c.Name, func(t *testing.T) {
			out, err := Load(c)
			if err != nil {
				t.Fatal(err)
			}
			for _, problem := range Judge(c, parse(out)) {
				t.Error(problem)
			}
		})
	}
}

// Judge says what is wrong with the failures a parse path made of a case; nothing when they keep the contract.
func Judge(c Case, failed []buildstep.Failure) []string {
	var problems []string
	say := func(s string) { problems = append(problems, c.Runner+" "+c.Name+": "+s) }
	if len(failed) != 1 {
		say(fmt.Sprintf("want one failure, got %d: %s", len(failed), describe(failed)))
		if len(failed) == 0 {
			return problems
		}
	}
	f := failed[0]
	if f.Test != c.Test {
		say("the failure names the test " + quote(f.Test) + ", want " + quote(c.Test))
	}
	if c.File != "" && f.File != c.File {
		say("the failure's file is " + quote(f.File) + ", want " + quote(c.File))
	}
	if why := buildstep.Hidden(f); why != "" {
		say("the failure hides its cause: " + why)
	}
	for _, want := range c.Cause {
		if !strings.Contains(f.Cause, want) {
			say("the cause " + quote(f.Cause) + " does not hold " + quote(want))
		}
	}
	if first := strings.SplitN(f.Output, "\n", 2)[0]; strings.TrimSpace(first) != strings.TrimSpace(f.Cause) {
		say("the output starts with " + quote(first) + ", not with the cause")
	}
	if len(f.Output) > buildstep.OutputBytes+buildstep.LineBytes+64 {
		say(fmt.Sprintf("the output holds %d bytes, over its bound", len(f.Output)))
	}
	for _, l := range strings.Split(f.Output, "\n") {
		if buildstep.PassingLine(l) {
			say("the output keeps the passing line " + quote(l))
		}
	}
	for _, p := range c.Passing {
		if strings.Contains(f.Output, p+"\n") || strings.HasSuffix(f.Output, p) || strings.Contains(f.Cause, p) || strings.Contains(f.Output, "- "+p) {
			say("the failure's words hold the passing test " + quote(p))
		}
	}
	return problems
}

func describe(failed []buildstep.Failure) string {
	var s []string
	for _, f := range failed {
		s = append(s, quote(f.Package+" "+f.Test)+" ("+quote(f.Cause)+")")
	}
	return strings.Join(s, ", ")
}

func quote(s string) string { return fmt.Sprintf("%q", s) }