novox/mesh-catalog
18
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases Wiki Activity

Build modules report failed tests through mesh-sdk's failure contract and run its conformance kit (issue 1901)

Browse Source 
mesh-build-node kept the first fifteen lines of node --test's output, so a passing line of the first file stood for the
failure. Both build modules now parse through the SDK (go test -json, node --test's TAP report written to a file, the
type-check and go vet as console failures), name themselves in the result, and run the SDK's captured cases; the
catalogue check failurekit refuses a node-build-* module with a test step whose tests do not run the kit.
This commit is contained in:
jschoubben
2026-10-26 19:48:39 +01:00
parent da29f77b10
commit 175b2cd791
29 changed files with 318 additions and 1313 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/mesh-build-node/cmd/mesh-build-node/check.go
+37 -24
View File
@@ -20,6 +20,7 @@ import (
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
@@ -54,14 +55,15 @@ type TestResult struct {
// the catalogue's TypeScript against the bounds is a later release's (R4f), not this one's gate.
OverBound bool `json:"over_bound"`
Failed []Failure `json:"failed,omitempty"`
// By is the build module and the SDK it was built with (buildstep.By): the checker names it when a failure hides
// its cause.
By string `json:"by,omitempty"`
}
// Failure is a check or script that failed; Output is the first lines it wrote.
type Failure struct {
Package string `json:"package"`
Test string `json:"test,omitempty"`
Output string `json:"output,omitempty"`
}
// Failure is the SDK's failure contract (buildstep.Failure, issue 1901): a failing test of the test script, read from
// node's TAP reporter, or a type-check or build that failed, each with its cause (never empty, never a passing line) and
// its own output up to 4096 bytes, cause first.
type Failure = buildstep.Failure
func (m *Module) testTool() StepTool {
return StepTool{
@@ -104,7 +106,7 @@ func (m *Module) test(ctx context.Context, args map[string]any) (any, error) {
if err != nil {
return nil, err
}
res := &TestResult{Workspace: workspace, Module: module, Level: level}
res := &TestResult{Workspace: workspace, Module: module, Level: level, By: buildstep.By("mesh-build-node")}
script := "test"
if level == "integration" {
script = "test:integration"
@@ -128,12 +130,12 @@ func (m *Module) test(ctx context.Context, args map[string]any) (any, error) {
var check, run time.Duration
var failed []Failure
// a command that could not start is not a verdict: the step fails. One that ran and said no is.
ran := func(name string, r forgekey.Ran) error {
ran := func(name string, r forgekey.Ran, report string) error {
if r.Err != "" {
return fmt.Errorf("%s could not be run: status %d %s: %s", name, r.Status, r.Err, firstLine(r.Stderr))
}
if r.Status != 0 {
failed = append(failed, Failure{Package: where, Test: name, Output: lines(r.Stdout + "\n" + r.Stderr)})
failed = append(failed, testFailures(where, name, dir, r, report)...)
}
return nil
}
@@ -148,17 +150,23 @@ func (m *Module) test(ctx context.Context, args map[string]any) (any, error) {
start := m.now()
r := m.runIn(ctx, root, false, dir, nil, "node", line...)
check = m.now().Sub(start)
if err := ran(name, r); err != nil {
if err := ran(name, r, ""); err != nil {
return nil, err
}
res.Ran = append(res.Ran, name)
}
if hasScript && len(failed) == 0 {
start := m.now()
env := []string{"PATH=" + binDir(root) + ":" + pathOrDefault(), "npm_config_cache=" + filepath.Join(m.Toolchain.Paths.Cache, "npm")}
// node --test writes its TAP report to a file of the workspace (issue 1901), whatever the script prints itself
report := filepath.Join(root, ".mesh-node-test-"+level+".tap")
_ = os.Remove(report)
env := []string{"PATH=" + binDir(root) + ":" + pathOrDefault(), "npm_config_cache=" + filepath.Join(m.Toolchain.Paths.Cache, "npm"),
"NODE_OPTIONS=" + buildstep.NodeTestOptions(report)}
r := m.runIn(ctx, root, false, dir, env, "npm", scriptArgs(script)...)
run = m.now().Sub(start)
if err := ran(script, r); err != nil {
tap, _ := os.ReadFile(report)
_ = os.Remove(report)
if err := ran(script, r, string(tap)); err != nil {
return nil, err
}
res.Ran = append(res.Ran, script)
@@ -166,7 +174,7 @@ func (m *Module) test(ctx context.Context, args map[string]any) (any, error) {
res.Failed = failed
var why []string
for _, f := range failed {
why = append(why, fmt.Sprintf("%s failed in %s: %s", f.Test, f.Package, firstLine(f.Output)))
why = append(why, failureWhy(f))
}
if level == "unit" {
res.Elapsed, res.Bound = (check + run).Seconds(), UnitBound.Seconds()
@@ -188,18 +196,23 @@ func (m *Module) test(ctx context.Context, args map[string]any) (any, error) {
return res, nil
}
// lines is the first lines of a command's output.
func lines(out string) string {
var keep []string
for _, l := range strings.Split(out, "\n") {
if l = strings.TrimSpace(l); l != "" {
keep = append(keep, l)
if len(keep) == 15 {
break
}
}
// testFailures is the module's parse path of a command that failed (the conformance kit runs it, issue 1901): the test
// script's failing tests from node's TAP report, or, for the type-check or the build, or a script whose report names no
// failing test, the command's own output with the line that names its error as the cause.
func testFailures(where, name, dir string, r forgekey.Ran, report string) []Failure {
if name == "test" || name == "test:integration" {
return buildstep.ReadTAP(where, name, dir, report, r.Stdout+"\n"+r.Stderr)
}
return strings.Join(keep, "\n")
return []Failure{buildstep.ConsoleFailure(where, name, r.Stdout+"\n"+r.Stderr)}
}
// failureWhy is the words of one failure: the test or check that failed, where, and its cause.
func failureWhy(f Failure) string {
file := ""
if f.File != "" {
file = " (" + f.File + ")"
}
return fmt.Sprintf("%s failed in %s%s: %s", f.Test, f.Package, file, firstLine(f.Cause))
}
func firstLine(s string) string {
modules/mesh-build-node/cmd/mesh-build-node/check_test.go
+5
View File
@@ -9,6 +9,7 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
@@ -198,4 +199,8 @@ func TestTheTestScriptFindsTscAndEsbuildOnItsPath(t *testing.T) {
if npm.name != "npm" || !strings.HasPrefix(path, bin+":") {
t.Errorf("npm ran with PATH %q, want %s first", path, bin)
}
// 1901/AC7: the test script runs under node's TAP reporter, written to a file of the workspace the step reads
if !slices.Contains(npm.env, "NODE_OPTIONS="+buildstep.NodeTestOptions(filepath.Join(work, "w1", ".mesh-node-test-unit.tap"))) {
t.Errorf("npm ran with %v, want NODE_OPTIONS naming the TAP reporter and its file", npm.env)
}
}
modules/mesh-build-node/cmd/mesh-build-node/kit_test.go
+20
View File
@@ -0,0 +1,20 @@
package main
import (
"testing"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/buildstep/conformance"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
// 1901/AC7: the module's own parse path (testFailures, as the test step calls it on a command that failed) keeps the
// failure contract on every captured node --test and tsc case of the SDK's conformance kit.
func TestTheTestStepKeepsTheFailureContract(t *testing.T) {
conformance.Run(t, conformance.NodeTest, func(o conformance.Output) []buildstep.Failure {
return testFailures(".", "test", o.Dir, forgekey.Ran{Status: 1, Stdout: o.Stdout, Stderr: o.Stderr}, o.Report)
})
conformance.Run(t, conformance.TypeCheck, func(o conformance.Output) []buildstep.Failure {
return testFailures(".", "type-check", o.Dir, forgekey.Ran{Status: 2, Stdout: o.Stdout, Stderr: o.Stderr}, "")
})
}
modules/mesh-build-node/go.mod
+1 -1
View File
@@ -3,7 +3,7 @@ module meshbuildnode
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026184535-e40dd0f56a78
github.com/distribution/distribution/v3 v3.1.2
github.com/nats-io/nats.go v1.54.0
)
modules/mesh-build-node/go.sum
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f h1:GYLCvz2x+JNUOTbXzyh2QwzX0sTzvRlXR7lGocWjk3A=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026184535-e40dd0f56a78 h1:7CdODXGPk15cuLyQfXI8HbiSbUYIvNAa1hwQ8laYxPE=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026184535-e40dd0f56a78/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI=
github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=