Build modules report failed tests through mesh-sdk's failure contract and run its conformance kit (issue 1901)
mesh-build-node kept the first fifteen lines of node --test's output, so a passing line of the first file stood for the failure. Both build modules now parse through the SDK (go test -json, node --test's TAP report written to a file, the type-check and go vet as console failures), name themselves in the result, and run the SDK's captured cases; the catalogue check failurekit refuses a node-build-* module with a test step whose tests do not run the kit.
This commit is contained in:
@@ -0,0 +1,115 @@
|
||||
// Package failurekit refuses a build module whose tests do not run mesh-sdk's conformance kit (issue 1901, its thorough
|
||||
// fix): a module that claims a node-build-* seat and serves a test step must hold a test that calls conformance.Run, so
|
||||
// its parse of the test runner's output is held to the failure contract (a failing test named with its cause, never a
|
||||
// passing line in its place). A build module with no test step (mesh-build-image builds images and runs no tests) is
|
||||
// exempt, and said so.
|
||||
package failurekit
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Judged is what the check says of one build module: Refused says why it fails the check, "" when it passes; Exempt
|
||||
// says why the kit does not apply.
|
||||
type Judged struct {
|
||||
Module, Refused, Exempt string
|
||||
}
|
||||
|
||||
type manifest struct {
|
||||
Claims []struct {
|
||||
Name string `json:"name"`
|
||||
} `json:"claims"`
|
||||
Tools []json.RawMessage `json:"tools"`
|
||||
}
|
||||
|
||||
// Judge reads every modules/<m>/module.json of the catalogue at root and judges each module that claims a node-build-*
|
||||
// seat.
|
||||
func Judge(root string) ([]Judged, error) {
|
||||
manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(manifests)
|
||||
var out []Judged
|
||||
for _, p := range manifests {
|
||||
raw, err := os.ReadFile(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var m manifest
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", p, err)
|
||||
}
|
||||
seat := ""
|
||||
for _, c := range m.Claims {
|
||||
if strings.HasPrefix(c.Name, "node-build-") {
|
||||
seat = c.Name
|
||||
}
|
||||
}
|
||||
if seat == "" {
|
||||
continue
|
||||
}
|
||||
dir := filepath.Dir(p)
|
||||
j := Judged{Module: filepath.Base(dir)}
|
||||
test := testTool(m.Tools)
|
||||
switch {
|
||||
case test == "":
|
||||
j.Exempt = fmt.Sprintf("it claims %s and serves no test step", seat)
|
||||
default:
|
||||
runs, err := runsTheKit(dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !runs {
|
||||
j.Refused = fmt.Sprintf("it claims %s and serves the test step %s, and no test of it calls conformance.Run "+
|
||||
"(mesh-sdk go/buildstep/conformance): run the kit with the module's own parse path (issue 1901)", seat, test)
|
||||
}
|
||||
}
|
||||
out = append(out, j)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// testTool is the name of the module's test step tool (<prefix>_test), or "".
|
||||
func testTool(tools []json.RawMessage) string {
|
||||
for _, raw := range tools {
|
||||
var name string
|
||||
if json.Unmarshal(raw, &name) != nil {
|
||||
var t struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
if json.Unmarshal(raw, &t) != nil {
|
||||
continue
|
||||
}
|
||||
name = t.Name
|
||||
}
|
||||
if strings.HasSuffix(name, "_test") {
|
||||
return name
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// runsTheKit says a _test.go file under dir calls conformance.Run.
|
||||
func runsTheKit(dir string) (bool, error) {
|
||||
found := false
|
||||
err := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil || found || d.IsDir() || !strings.HasSuffix(p, "_test.go") {
|
||||
return err
|
||||
}
|
||||
raw, err := os.ReadFile(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
found = strings.Contains(string(raw), "conformance.Run(") &&
|
||||
strings.Contains(string(raw), `"git.novox.be/novox/mesh-sdk/go/buildstep/conformance"`)
|
||||
return nil
|
||||
})
|
||||
return found, err
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package failurekit
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// 1901/AC7: a build module with a test step and no kit is refused, one that runs the kit passes, one with no test step is
|
||||
// exempt, and a module that claims no node-build-* seat is not judged.
|
||||
func TestABuildModuleWithATestStepMustRunTheKit(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
kit := "package main\n\nimport \"git.novox.be/novox/mesh-sdk/go/buildstep/conformance\"\n\nfunc TestKit(t *testing.T) { conformance.Run(t, conformance.GoTest, nil) }\n"
|
||||
for path, body := range map[string]string{
|
||||
"modules/a/module.json": `{"claims":[{"name":"node-build-a"}],"tools":["build_a_fetch","build_a_test"]}`,
|
||||
"modules/a/cmd/a/check_test.go": "package main\n",
|
||||
"modules/b/module.json": `{"claims":[{"name":"node-build-b"}],"tools":[{"name":"build_b_test"}]}`,
|
||||
"modules/b/cmd/b/kit_test.go": kit,
|
||||
"modules/c/module.json": `{"claims":[{"name":"node-build-c"}],"tools":["build_c_build"]}`,
|
||||
"modules/d/module.json": `{"claims":[{"name":"other"}],"tools":["d_test"]}`,
|
||||
"modules/e/module.json": `{"claims":[{"name":"node-build-e"}],"tools":["build_e_test"]}`,
|
||||
"modules/e/cmd/e/kit.go": kit, // not a test file
|
||||
} {
|
||||
p := filepath.Join(root, path)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got, err := Judge(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := map[string]string{"a": "refused", "b": "passes", "c": "exempt", "e": "refused"}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("judged %+v, want a, b, c and e", got)
|
||||
}
|
||||
for _, j := range got {
|
||||
is := "passes"
|
||||
if j.Refused != "" {
|
||||
is = "refused"
|
||||
} else if j.Exempt != "" {
|
||||
is = "exempt"
|
||||
}
|
||||
if is != want[j.Module] {
|
||||
t.Errorf("%s %s (%+v), want it %s", j.Module, is, j, want[j.Module])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Every build module of this catalogue with a test step runs the kit.
|
||||
func TestTheCataloguesBuildModulesRunTheKit(t *testing.T) {
|
||||
got, err := Judge(filepath.Join("..", ".."))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) == 0 {
|
||||
t.Fatal("no module claims a node-build-* seat: the check reads the wrong tree")
|
||||
}
|
||||
for _, j := range got {
|
||||
switch {
|
||||
case j.Refused != "":
|
||||
t.Errorf("%s: %s", j.Module, j.Refused)
|
||||
case j.Exempt != "":
|
||||
t.Logf("%s is exempt: %s", j.Module, j.Exempt)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
module git.novox.be/novox/mesh-catalog/checks/failurekit
|
||||
|
||||
go 1.22
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
Reference in New Issue
Block a user