mesh-lab/replays/cmd/prove/main.go

// prove runs each replay on the commit before its fix and on its fix, and says whether it fails before
// and passes after (novox/hq to-be 45 Phase 5, "done when"). A replay that passes before its fix proves
// nothing about it, and one that fails after it is not a replay of that fix.
//
//	go run ./cmd/prove [--repos <dir holding the clones, ../.. by default>] [R262 R266 …]
//
// The clones are the core repositories, side by side as the lab expects them. A replay in its repository
// is laid over a worktree of each commit and run with go test; the bus replay runs against a server of
// the release the catalogue pinned at each commit, raised for it; the resolver replay with the catalogue's
// resolver at each commit. MESH_TEST_POSTGRES is passed through to the controller's.
package main

import (
	"context"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/novox/mesh-lab/replays"
)

func main() {
	repos := flag.String("repos", filepath.Join("..", ".."), "the directory the core repositories are cloned side by side in")
	flag.Parse()
	if abs, err := filepath.Abs(*repos); err == nil {
		*repos = abs
	}
	want := flag.Args()
	failed := 0
	for _, r := range replays.Register {
		if len(want) > 0 && !contains(want, r.ID) {
			continue
		}
		at := r.Fix + "^1"
		if r.Before != "" {
			at = r.Before
		}
		before, beforeSaid := run(r, *repos, at, false)
		after, afterSaid := run(r, *repos, r.Fix, true)
		verdict := "PROVED"
		if before != "fails" && before != "absent" || after != "passes" {
			verdict = "NOT PROVED"
			failed++
		}
		fmt.Printf("%s (issue %d): %s — before its fix %s, on its fix %s\n", r.ID, r.Issue, verdict, before, after)
		if verdict != "PROVED" {
			fmt.Printf("  before: %s\n  after: %s\n", beforeSaid, afterSaid)
		} else {
			fmt.Printf("  before: %s\n", firstLine(beforeSaid))
		}
	}
	if failed > 0 {
		os.Exit(1)
	}
}

// run is the replay at one commit: passes, fails, or absent (the check it replays did not exist).
func run(r replays.Replay, repos, at string, fixed bool) (string, string) {
	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
	defer cancel()
	repo := filepath.Join(repos, r.Repository)
	tree, err := os.MkdirTemp("", "prove-"+r.ID+"-")
	if err != nil {
		return "error", err.Error()
	}
	defer os.RemoveAll(tree)
	if out, err := exec.CommandContext(ctx, "git", "-C", repo, "worktree", "add", "--quiet", "--detach", tree, at).CombinedOutput(); err != nil {
		return "error", string(out)
	}
	defer exec.Command("git", "-C", repo, "worktree", "remove", "--force", tree).Run()

	switch r.Kind {
	case replays.InRepository, replays.Gate, replays.Liveness:
		home := filepath.Join(repos, r.Repository)
		if r.Home != "" {
			home = filepath.Join(repos, r.Home)
		}
		ref := r.HomeRef
		if ref == "" {
			ref = r.Fix
		}
		for _, f := range r.Files {
			body, err := exec.Command("git", "-C", home, "show", ref+":"+f).Output()
			if err != nil {
				return "error", fmt.Sprintf("%s has no %s: %v", ref, f, err)
			}
			if err := os.WriteFile(filepath.Join(tree, f), body, 0o644); err != nil {
				return "error", err.Error()
			}
		}
		if r.Kind == replays.Liveness {
			// The other repositories at their own commits on the same side, and the replay here run
			// against all of them.
			env := []string{"MESH_REPLAY_CONTROLLER=" + tree}
			for _, also := range r.With {
				ref := also.Fix
				if !fixed {
					ref = also.Fix + "^1"
					if also.Before != "" {
						ref = also.Before
					}
				}
				other, err := os.MkdirTemp("", "prove-"+r.ID+"-"+also.Repository+"-")
				if err != nil {
					return "error", err.Error()
				}
				defer os.RemoveAll(other)
				clone := filepath.Join(repos, also.Repository)
				if out, err := exec.CommandContext(ctx, "git", "-C", clone, "worktree", "add", "--quiet", "--detach", other,
					ref).CombinedOutput(); err != nil {
					return "error", string(out)
				}
				defer exec.Command("git", "-C", clone, "worktree", "remove", "--force", other).Run()
				if also.Repository == "mesh-host" {
					env = append(env, "MESH_REPLAY_HOST="+other)
				}
			}
			cmd := exec.CommandContext(ctx, "go", "test", "-count=1", "-run", "^"+r.Test+"$", r.Package)
			cmd.Env = append(os.Environ(), env...)
			out, err := cmd.CombinedOutput()
			return verdictOf(string(out), err, false)
		}
		mode := "-mod=vendor"
		if _, err := os.Stat(filepath.Join(tree, r.Module, "vendor")); err != nil {
			mode = "-mod=mod"
		}
		cmd := exec.CommandContext(ctx, "go", "test", "-count=1", "-run", "^"+r.Test, r.Package)
		cmd.Dir = filepath.Join(tree, r.Module)
		cmd.Env = append(os.Environ(), "GOFLAGS="+mode, "GOPRIVATE=git.novox.be")
		out, err := cmd.CombinedOutput()
		return verdictOf(string(out), err, r.Kind == replays.Gate)
	case replays.Bus:
		// **The very image the catalogue pinned at the commit**, by its digest: the release a digest is was
		// not said beside it before the fix, and the digest is what ran.
		image, err := pinnedBus(tree)
		if err != nil {
			return "error", err.Error()
		}
		url, stop, err := aBus(ctx, image)
		if err != nil {
			return "error", err.Error()
		}
		defer stop()
		return replayHere(ctx, r, "MESH_TEST_NATS="+url)
	case replays.Resolver:
		return replayHere(ctx, r, "MESH_REPLAY_CATALOGUE="+tree)
	}
	return "error", "no such kind"
}

func replayHere(ctx context.Context, r replays.Replay, env string) (string, string) {
	cmd := exec.CommandContext(ctx, "go", "test", "-count=1", "-run", "^"+r.Test, r.Package)
	cmd.Env = append(os.Environ(), env)
	out, err := cmd.CombinedOutput()
	return verdictOf(string(out), err, false)
}

// verdictOf reads go test's word: a replay that does not build where the check did not exist is absent.
func verdictOf(out string, err error, gate bool) (string, string) {
	switch {
	case err == nil && strings.Contains(out, "no tests to run"):
		return "absent", "no test of that name at this commit"
	case err == nil:
		return "passes", lastLines(out, 3)
	case strings.Contains(out, "[build failed]") || strings.Contains(out, "[setup failed]"):
		if gate {
			return "absent", "the check did not exist: " + lastLines(out, 3)
		}
		return "error", "does not build: " + lastLines(out, 6)
	case strings.Contains(out, "--- FAIL"):
		return "fails", failure(out)
	}
	return "error", lastLines(out, 6)
}

// pinnedBus is the bus image the catalogue's nats module is built on at a commit.
func pinnedBus(tree string) (string, error) {
	raw, err := os.ReadFile(filepath.Join(tree, "modules", "nats", "module.json"))
	if err != nil {
		return "", err
	}
	var m struct {
		Build struct {
			On []struct{ Arg, Image string } `json:"on"`
		} `json:"build"`
	}
	if err := json.Unmarshal(raw, &m); err != nil {
		return "", err
	}
	for _, on := range m.Build.On {
		if on.Arg == "NATS_BASE" && on.Image != "" {
			return on.Image, nil
		}
	}
	return "", errors.New("the catalogue's nats module names no NATS_BASE image")
}

// aBus is a server of an image, on loopback, for the length of one replay.
func aBus(ctx context.Context, image string) (string, func(), error) {
	name := fmt.Sprintf("prove-bus-%d", time.Now().UnixNano())
	if out, err := exec.CommandContext(ctx, "docker", "run", "-d", "--rm", "--name", name, "-p", "127.0.0.1::4222",
		image, "-js").CombinedOutput(); err != nil {
		return "", nil, fmt.Errorf("%s: %v %s", image, err, out)
	}
	stop := func() { _ = exec.Command("docker", "rm", "-f", name).Run() }
	out, err := exec.CommandContext(ctx, "docker", "port", name, "4222/tcp").Output()
	if err != nil {
		stop()
		return "", nil, err
	}
	address := strings.TrimSpace(strings.Split(string(out), "\n")[0])
	for i := 0; i < 50; i++ {
		if c, err := net.Dial("tcp", address); err == nil {
			c.Close()
			break
		}
		time.Sleep(200 * time.Millisecond)
	}
	return "nats://" + address, stop, nil
}

func failure(out string) string {
	for _, line := range strings.Split(out, "\n") {
		if strings.Contains(line, "_test.go:") {
			return strings.TrimSpace(line)
		}
	}
	return lastLines(out, 3)
}

func lastLines(s string, n int) string {
	lines := strings.Split(strings.TrimSpace(s), "\n")
	if len(lines) > n {
		lines = lines[len(lines)-n:]
	}
	return strings.Join(lines, " | ")
}

func firstLine(s string) string {
	line, _, _ := strings.Cut(s, "\n")
	if len(line) > 300 {
		line = line[:300] + "…"
	}
	return line
}

func contains(list []string, s string) bool {
	for _, x := range list {
		if x == s {
			return true
		}
	}
	return false
}