mesh-catalog/checks/failurekit/failurekit.go

// Package failurekit refuses a build module whose tests do not run the build runner's conformance kit (issue 1901, its thorough
// fix): a module that claims a node-build-* seat and serves a test step must hold a test that calls conformance.Run, so
// its parse of the test runner's output is held to the failure contract (a failing test named with its cause, never a
// passing line in its place). A build module with no test step (mesh-build-image builds images and runs no tests) is
// exempt from that, and said so. Every build module, test step or not, must also hold a test that calls
// conformance.LeftRunning with its own start sweep, so a step a previous process left running is ended at its start
// (issue 1890), and one that calls conformance.Passing, so a step whose network did not answer is said as a transport fault
// (issue 1748). Its manifest asks the capability seat-settle-delay (ADR 0664 §5).
package failurekit

import (
	"encoding/json"
	"fmt"
	"io/fs"
	"os"
	"path/filepath"
	"slices"
	"sort"
	"strings"
)

// Judged is what the check says of one build module: Refused says why it fails the check, "" when it passes; Exempt
// says why the kit does not apply.
type Judged struct {
	Module, Refused, Exempt string
}

type manifest struct {
	Claims []struct {
		Name string `json:"name"`
	} `json:"claims"`
	Tools        []json.RawMessage `json:"tools"`
	Capabilities []string          `json:"capabilities"`
}

// settleDelay is the capability a tool runner states when it can settle a seat's ask with a delayed negative acknowledgement
// (mesh-sdk CapSeatSettleDelay): a build module gives its ask again after a delay by it, so the delivery plan places a tool
// runner that states it before the module (ADR 0664 §1, §5).
const settleDelay = "seat-settle-delay"

// Judge reads every modules/<m>/module.json of the catalogue at root and judges each module that claims a node-build-*
// seat.
func Judge(root string) ([]Judged, error) {
	manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
	if err != nil {
		return nil, err
	}
	sort.Strings(manifests)
	var out []Judged
	for _, p := range manifests {
		raw, err := os.ReadFile(p)
		if err != nil {
			return nil, err
		}
		var m manifest
		if err := json.Unmarshal(raw, &m); err != nil {
			return nil, fmt.Errorf("%s: %w", p, err)
		}
		seat := ""
		for _, c := range m.Claims {
			if strings.HasPrefix(c.Name, "node-build-") {
				seat = c.Name
			}
		}
		if seat == "" {
			continue
		}
		dir := filepath.Dir(p)
		j := Judged{Module: filepath.Base(dir)}
		var refused []string
		test := testTool(m.Tools)
		switch {
		case test == "":
			j.Exempt = fmt.Sprintf("it claims %s and serves no test step, so the failure contract's cases do not apply", seat)
		default:
			runs, err := runsTheKit(dir, "conformance.Run(")
			if err != nil {
				return nil, err
			}
			if !runs {
				refused = append(refused, fmt.Sprintf("it claims %s and serves the test step %s, and no test of it calls conformance.Run "+
					"(mesh-build-go buildstep/conformance): run the kit with the module's own parse path (issue 1901)", seat, test))
			}
		}
		// Every build module keeps steps, so every one runs the left-running case, test step or not (issue 1890).
		sweeps, err := runsTheKit(dir, "conformance.LeftRunning(")
		if err != nil {
			return nil, err
		}
		if !sweeps {
			refused = append(refused, fmt.Sprintf("it claims %s, and no test of it calls conformance.LeftRunning "+
				"(mesh-build-go buildstep/conformance): run the kit's left-running case with the module's own start sweep (issue 1890)", seat))
		}
		// Every build module meets the network (a fetch, a download, an install, a publish), so every one runs the passing case: its
		// steps say a network that did not answer as a transport fault the bus gives the ask again for (ADR 0664, issue 1748).
		says, err := runsTheKit(dir, "conformance.Passing(")
		if err != nil {
			return nil, err
		}
		if !says {
			refused = append(refused, fmt.Sprintf("it claims %s, and no test of it calls conformance.Passing "+
				"(mesh-build-go buildstep/conformance): run the kit's passing case with the module's own path from a failed run to its step's error (issue 1748)", seat))
		}
		// Every build module gives a step that met a transport fault again by the bus (ADR 0664 §5): it asks the tool runner's
		// capability in its manifest, so the delivery plan places a tool runner that states it first.
		if !slices.Contains(m.Capabilities, settleDelay) {
			refused = append(refused, fmt.Sprintf("it claims %s, and its manifest's capabilities do not ask %s: a step given again by the bus needs "+
				"a tool runner that can settle with a delay (ADR 0664 §5, issue 1748)", seat, settleDelay))
		}
		j.Refused = strings.Join(refused, "; ")
		out = append(out, j)
	}
	return out, nil
}

// testTool is the name of the module's test step tool (<prefix>_test), or "".
func testTool(tools []json.RawMessage) string {
	for _, raw := range tools {
		var name string
		if json.Unmarshal(raw, &name) != nil {
			var t struct {
				Name string `json:"name"`
			}
			if json.Unmarshal(raw, &t) != nil {
				continue
			}
			name = t.Name
		}
		if strings.HasSuffix(name, "_test") {
			return name
		}
	}
	return ""
}

// runsTheKit says a _test.go file under dir calls the kit's function call (such as "conformance.Run(").
func runsTheKit(dir, call string) (bool, error) {
	found := false
	err := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
		if err != nil || found || d.IsDir() || !strings.HasSuffix(p, "_test.go") {
			return err
		}
		raw, err := os.ReadFile(p)
		if err != nil {
			return err
		}
		found = strings.Contains(string(raw), call) &&
			strings.Contains(string(raw), `"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep/conformance"`)
		return nil
	})
	return found, err
}