mesh-lab/replays/cmd/prove/main.go

// prove runs each replay on the commit before its fix and on its fix, and says whether it fails before
// and passes after (novox/hq to-be 45 Phase 5, "done when"). A replay that passes before its fix proves
// nothing about it, and one that fails after it is not a replay of that fix.
//
//	go run ./cmd/prove [--repos <dir holding the clones, ../.. by default>] [R262 R266 …]
//
// The clones are the core repositories, side by side as the lab expects them. A replay in its repository
// is laid over a worktree of each commit and run with go test; the bus replay runs against a server of
// the release the catalogue pinned at each commit, raised for it; the resolver replay with the catalogue's
// resolver at each commit; a bed of the lab is raised on its hypervisor with the node-engine and its
// installer built at each commit (a workstation where `mesh-lab check` passes). MESH_TEST_POSTGRES is passed
// through to the controller's.
package main

import (
	"context"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/novox/mesh-lab/replays"
)

func main() {
	repos := flag.String("repos", filepath.Join("..", ".."), "the directory the core repositories are cloned side by side in")
	flag.Parse()
	if abs, err := filepath.Abs(*repos); err == nil {
		*repos = abs
	}
	want := flag.Args()
	failed := 0
	for _, r := range replays.Register {
		if len(want) > 0 && !contains(want, r.ID) {
			continue
		}
		at := r.Fix + "^1"
		if r.Before != "" {
			at = r.Before
		}
		before, beforeSaid := run(r, *repos, at, false)
		after, afterSaid := run(r, *repos, r.Fix, true)
		verdict := "PROVED"
		if before != "fails" && before != "absent" || after != "passes" {
			verdict = "NOT PROVED"
			failed++
		}
		fmt.Printf("%s (issue %d): %s — before its fix %s, on its fix %s\n", r.ID, r.Issue, verdict, before, after)
		if verdict != "PROVED" {
			fmt.Printf("  before: %s\n  after: %s\n", beforeSaid, afterSaid)
		} else {
			fmt.Printf("  before: %s\n", firstLine(beforeSaid))
		}
	}
	if failed > 0 {
		os.Exit(1)
	}
}

// run is the replay at one commit: passes, fails, or absent (the check it replays did not exist).
func run(r replays.Replay, repos, at string, fixed bool) (string, string) {
	limit := 15 * time.Minute
	if r.Kind == replays.Bed {
		// Two machines raised, a foundation pulled and a control plane built inside one of them.
		limit = 90 * time.Minute
	}
	ctx, cancel := context.WithTimeout(context.Background(), limit)
	defer cancel()
	repo := filepath.Join(repos, r.Repository)
	tree, err := os.MkdirTemp("", "prove-"+r.ID+"-")
	if err != nil {
		return "error", err.Error()
	}
	defer os.RemoveAll(tree)
	if out, err := exec.CommandContext(ctx, "git", "-C", repo, "worktree", "add", "--quiet", "--detach", tree, at).CombinedOutput(); err != nil {
		return "error", string(out)
	}
	defer exec.Command("git", "-C", repo, "worktree", "remove", "--force", tree).Run()

	switch r.Kind {
	case replays.Bed:
		return aBed(ctx, r, repos, tree, fixed)
	case replays.InRepository, replays.Gate, replays.Liveness:
		home := filepath.Join(repos, r.Repository)
		if r.Home != "" {
			home = filepath.Join(repos, r.Home)
		}
		ref := r.HomeRef
		if ref == "" {
			ref = r.Fix
		}
		for _, f := range r.Files {
			body, err := exec.Command("git", "-C", home, "show", ref+":"+f).Output()
			if err != nil {
				return "error", fmt.Sprintf("%s has no %s: %v", ref, f, err)
			}
			if err := os.WriteFile(filepath.Join(tree, f), body, 0o644); err != nil {
				return "error", err.Error()
			}
		}
		if r.Kind == replays.Liveness {
			// The other repositories at their own commits on the same side, and the replay here run
			// against all of them.
			env := []string{"MESH_REPLAY_CONTROLLER=" + tree}
			for _, also := range r.With {
				ref := also.Fix
				if !fixed {
					ref = also.Fix + "^1"
					if also.Before != "" {
						ref = also.Before
					}
				}
				other, err := os.MkdirTemp("", "prove-"+r.ID+"-"+also.Repository+"-")
				if err != nil {
					return "error", err.Error()
				}
				defer os.RemoveAll(other)
				clone := filepath.Join(repos, also.Repository)
				if out, err := exec.CommandContext(ctx, "git", "-C", clone, "worktree", "add", "--quiet", "--detach", other,
					ref).CombinedOutput(); err != nil {
					return "error", string(out)
				}
				defer exec.Command("git", "-C", clone, "worktree", "remove", "--force", other).Run()
				if also.Repository == "mesh-host" {
					env = append(env, "MESH_REPLAY_HOST="+other)
				}
			}
			cmd := exec.CommandContext(ctx, "go", "test", "-count=1", "-run", "^"+r.Test+"$", r.Package)
			cmd.Env = append(os.Environ(), env...)
			out, err := cmd.CombinedOutput()
			return verdictOf(string(out), err, false)
		}
		mode := "-mod=vendor"
		if _, err := os.Stat(filepath.Join(tree, r.Module, "vendor")); err != nil {
			mode = "-mod=mod"
		}
		cmd := exec.CommandContext(ctx, "go", "test", "-count=1", "-run", "^"+r.Test, r.Package)
		cmd.Dir = filepath.Join(tree, r.Module)
		cmd.Env = append(os.Environ(), "GOFLAGS="+mode, "GOPRIVATE=git.novox.be")
		out, err := cmd.CombinedOutput()
		return verdictOf(string(out), err, r.Kind == replays.Gate)
	case replays.Bus:
		// **The very image the catalogue pinned at the commit**, by its digest: the release a digest is was
		// not said beside it before the fix, and the digest is what ran.
		image, err := pinnedBus(tree)
		if err != nil {
			return "error", err.Error()
		}
		url, stop, err := aBus(ctx, image)
		if err != nil {
			return "error", err.Error()
		}
		defer stop()
		return replayHere(ctx, r, "MESH_TEST_NATS="+url)
	case replays.Resolver:
		return replayHere(ctx, r, "MESH_REPLAY_CATALOGUE="+tree)
	}
	return "error", "no such kind"
}

// aBed raises the lab's bed at one commit of the fixed repository (novox/hq issue 92): the node-engine and its
// installer built from it, the installer carrying a builder built from the controller the replay pins, and
// the bed run with them, the pinned controller to build from and the pinned catalogue to read.
func aBed(ctx context.Context, r replays.Replay, repos, tree string, fixed bool) (string, string) {
	beside := map[string]string{}
	for _, also := range r.With {
		ref := also.Fix
		if !fixed {
			ref = also.Fix + "^1"
			if also.Before != "" {
				ref = also.Before
			}
		}
		other, err := os.MkdirTemp("", "prove-"+r.ID+"-"+also.Repository+"-")
		if err != nil {
			return "error", err.Error()
		}
		defer os.RemoveAll(other)
		clone := filepath.Join(repos, also.Repository)
		if out, err := exec.CommandContext(ctx, "git", "-C", clone, "worktree", "add", "--quiet", "--detach", other,
			ref).CombinedOutput(); err != nil {
			return "error", string(out)
		}
		defer exec.Command("git", "-C", clone, "worktree", "remove", "--force", other).Run()
		beside[also.Repository] = other
	}
	controller, catalogue := beside["mesh-controller"], beside["mesh-catalog"]
	if controller == "" || catalogue == "" {
		return "error", "a bed is raised with the controller and the catalogue it pins, and this one pins " +
			"not both"
	}
	builder, err := builderImage(ctx, controller)
	if err != nil {
		return "error", err.Error()
	}
	host, installer := filepath.Join(tree, "mesh-host"), filepath.Join(tree, "mesh-bootstrap")
	for _, step := range [][]string{
		{"make", "build", "SYSTEM=arch"},
		{"make", "bootstrap", "IMAGE=" + builder, "BOOTSTRAP_OUT=" + installer},
	} {
		cmd := exec.CommandContext(ctx, step[0], step[1:]...)
		cmd.Dir = tree
		if out, err := cmd.CombinedOutput(); err != nil {
			return "error", fmt.Sprintf("%s: %v %s", strings.Join(step, " "), err, lastLines(string(out), 6))
		}
	}
	lab, err := filepath.Abs("..")
	if err != nil {
		return "error", err.Error()
	}
	cmd := exec.CommandContext(ctx, "node", "--test", "--test-reporter=tap", "--experimental-strip-types", r.Test)
	cmd.Dir = lab
	cmd.Env = append(os.Environ(),
		"MESH_LAB_HOST_BINARY="+host,
		"MESH_LAB_BOOTSTRAP_BINARY="+installer,
		// The template the installer knows how to make safe: it rotates the store's and the broker's
		// well-known credentials, and refuses a template naming neither (the bus one names no broker).
		"MESH_LAB_BUNDLE="+filepath.Join(tree, "examples", "foundation-first-node.lock"),
		"MESH_LAB_CATALOG="+filepath.Join(catalogue, "modules"),
		"MESH_REPLAY_CONTROLLER="+controller)
	out, err := cmd.CombinedOutput()
	return bedVerdict(string(out), err)
}

// builderImage is the builder the installer carries, built from the controller at the commit the replay
// pins, on the Go release its go.mod asks for; built once and kept under a tag naming the commit.
func builderImage(ctx context.Context, controller string) (string, error) {
	head, err := exec.CommandContext(ctx, "git", "-C", controller, "rev-parse", "--short=12", "HEAD").Output()
	if err != nil {
		return "", err
	}
	tag := "mesh-builder:prove-" + strings.TrimSpace(string(head))
	if exec.CommandContext(ctx, "docker", "image", "inspect", tag).Run() == nil {
		return tag, nil
	}
	mod, err := os.ReadFile(filepath.Join(controller, "go.mod"))
	if err != nil {
		return "", err
	}
	release := ""
	for _, line := range strings.Split(string(mod), "\n") {
		if v, ok := strings.CutPrefix(line, "go "); ok {
			parts := strings.Split(strings.TrimSpace(v), ".")
			if len(parts) >= 2 {
				release = parts[0] + "." + parts[1]
			}
		}
	}
	if release == "" {
		return "", errors.New("the controller's go.mod names no Go release to build its builder on")
	}
	cmd := exec.CommandContext(ctx, "docker", "build", "-q", "--build-arg", "GO_BASE=golang:"+release+"-alpine",
		"-f", "cmd/mesh-builder/Dockerfile", "-t", tag, ".")
	cmd.Dir = controller
	if out, err := cmd.CombinedOutput(); err != nil {
		return "", fmt.Errorf("the builder could not be built from %s: %v %s", controller, err, lastLines(string(out), 6))
	}
	return tag, nil
}

// bedVerdict reads node's test runner: a skipped bed is an error — it never reached what it replays, or was
// not given what it raises — and never the fault reproduced.
func bedVerdict(out string, err error) (string, string) {
	count := func(what string) int {
		m := regexp.MustCompile(`(?m)^# ` + what + ` (\d+)$`).FindStringSubmatch(out)
		if m == nil {
			return -1
		}
		n, _ := strconv.Atoi(m[1])
		return n
	}
	pass, fail, skipped := count("pass"), count("fail"), count("skipped")
	switch {
	case skipped > 0:
		said := regexp.MustCompile(`(?m)# SKIP (.*)$`).FindStringSubmatch(out)
		if said != nil {
			return "error", "the bed skipped: " + said[1]
		}
		return "error", "the bed skipped: " + lastLines(out, 6)
	case fail > 0:
		if m := regexp.MustCompile(`(?m)^\s+error: [|>]?-?\s*\n?\s*(.+)$`).FindStringSubmatch(out); m != nil {
			return "fails", strings.TrimSpace(m[1])
		}
		return "fails", lastLines(out, 6)
	case pass > 0 && err == nil:
		return "passes", lastLines(out, 3)
	}
	return "error", lastLines(out, 6)
}

func replayHere(ctx context.Context, r replays.Replay, env string) (string, string) {
	cmd := exec.CommandContext(ctx, "go", "test", "-count=1", "-run", "^"+r.Test, r.Package)
	cmd.Env = append(os.Environ(), env)
	out, err := cmd.CombinedOutput()
	return verdictOf(string(out), err, false)
}

// verdictOf reads go test's word: a replay that does not build where the check did not exist is absent.
func verdictOf(out string, err error, gate bool) (string, string) {
	switch {
	case err == nil && strings.Contains(out, "no tests to run"):
		return "absent", "no test of that name at this commit"
	case err == nil:
		return "passes", lastLines(out, 3)
	case strings.Contains(out, "[build failed]") || strings.Contains(out, "[setup failed]"):
		if gate {
			return "absent", "the check did not exist: " + lastLines(out, 3)
		}
		return "error", "does not build: " + lastLines(out, 6)
	case strings.Contains(out, "--- FAIL"):
		return "fails", failure(out)
	}
	return "error", lastLines(out, 6)
}

// pinnedBus is the bus image the catalogue's nats module is built on at a commit.
func pinnedBus(tree string) (string, error) {
	raw, err := os.ReadFile(filepath.Join(tree, "modules", "nats", "module.json"))
	if err != nil {
		return "", err
	}
	var m struct {
		Build struct {
			On []struct{ Arg, Image string } `json:"on"`
		} `json:"build"`
	}
	if err := json.Unmarshal(raw, &m); err != nil {
		return "", err
	}
	for _, on := range m.Build.On {
		if on.Arg == "NATS_BASE" && on.Image != "" {
			return on.Image, nil
		}
	}
	return "", errors.New("the catalogue's nats module names no NATS_BASE image")
}

// aBus is a server of an image, on loopback, for the length of one replay.
func aBus(ctx context.Context, image string) (string, func(), error) {
	name := fmt.Sprintf("prove-bus-%d", time.Now().UnixNano())
	if out, err := exec.CommandContext(ctx, "docker", "run", "-d", "--rm", "--name", name, "-p", "127.0.0.1::4222",
		image, "-js").CombinedOutput(); err != nil {
		return "", nil, fmt.Errorf("%s: %v %s", image, err, out)
	}
	stop := func() { _ = exec.Command("docker", "rm", "-f", name).Run() }
	out, err := exec.CommandContext(ctx, "docker", "port", name, "4222/tcp").Output()
	if err != nil {
		stop()
		return "", nil, err
	}
	address := strings.TrimSpace(strings.Split(string(out), "\n")[0])
	for i := 0; i < 50; i++ {
		if c, err := net.Dial("tcp", address); err == nil {
			c.Close()
			break
		}
		time.Sleep(200 * time.Millisecond)
	}
	return "nats://" + address, stop, nil
}

func failure(out string) string {
	for _, line := range strings.Split(out, "\n") {
		if strings.Contains(line, "_test.go:") {
			return strings.TrimSpace(line)
		}
	}
	return lastLines(out, 3)
}

func lastLines(s string, n int) string {
	lines := strings.Split(strings.TrimSpace(s), "\n")
	if len(lines) > n {
		lines = lines[len(lines)-n:]
	}
	return strings.Join(lines, " | ")
}

func firstLine(s string) string {
	line, _, _ := strings.Cut(s, "\n")
	if len(line) > 300 {
		line = line[:300] + "…"
	}
	return line
}

func contains(list []string, s string) bool {
	for _, x := range list {
		if x == s {
			return true
		}
	}
	return false
}