// prove runs each replay on the commit before its fix and on its fix, and says whether it fails before // and passes after (novox/hq to-be 45 Phase 5, "done when"). A replay that passes before its fix proves // nothing about it, and one that fails after it is not a replay of that fix. // // go run ./cmd/prove [--repos ] [R262 R266 …] // // The clones are the core repositories, side by side as the lab expects them. A replay in its repository // is laid over a worktree of each commit and run with go test; the bus replay runs against a server of // the release the catalogue pinned at each commit, raised for it; the resolver replay with the catalogue's // resolver at each commit. MESH_TEST_POSTGRES is passed through to the controller's. package main import ( "context" "encoding/json" "errors" "flag" "fmt" "net" "os" "os/exec" "path/filepath" "strings" "time" "github.com/novox/mesh-lab/replays" ) func main() { repos := flag.String("repos", filepath.Join("..", ".."), "the directory the core repositories are cloned side by side in") flag.Parse() if abs, err := filepath.Abs(*repos); err == nil { *repos = abs } want := flag.Args() failed := 0 for _, r := range replays.Register { if len(want) > 0 && !contains(want, r.ID) { continue } at := r.Fix + "^1" if r.Before != "" { at = r.Before } before, beforeSaid := run(r, *repos, at, false) after, afterSaid := run(r, *repos, r.Fix, true) verdict := "PROVED" if before != "fails" && before != "absent" || after != "passes" { verdict = "NOT PROVED" failed++ } fmt.Printf("%s (issue %d): %s — before its fix %s, on its fix %s\n", r.ID, r.Issue, verdict, before, after) if verdict != "PROVED" { fmt.Printf(" before: %s\n after: %s\n", beforeSaid, afterSaid) } else { fmt.Printf(" before: %s\n", firstLine(beforeSaid)) } } if failed > 0 { os.Exit(1) } } // run is the replay at one commit: passes, fails, or absent (the check it replays did not exist). func run(r replays.Replay, repos, at string, fixed bool) (string, string) { ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) defer cancel() repo := filepath.Join(repos, r.Repository) tree, err := os.MkdirTemp("", "prove-"+r.ID+"-") if err != nil { return "error", err.Error() } defer os.RemoveAll(tree) if out, err := exec.CommandContext(ctx, "git", "-C", repo, "worktree", "add", "--quiet", "--detach", tree, at).CombinedOutput(); err != nil { return "error", string(out) } defer exec.Command("git", "-C", repo, "worktree", "remove", "--force", tree).Run() switch r.Kind { case replays.InRepository, replays.Gate, replays.Liveness: home := filepath.Join(repos, r.Repository) if r.Home != "" { home = filepath.Join(repos, r.Home) } ref := r.HomeRef if ref == "" { ref = r.Fix } for _, f := range r.Files { body, err := exec.Command("git", "-C", home, "show", ref+":"+f).Output() if err != nil { return "error", fmt.Sprintf("%s has no %s: %v", ref, f, err) } if err := os.WriteFile(filepath.Join(tree, f), body, 0o644); err != nil { return "error", err.Error() } } if r.Kind == replays.Liveness { // The other repositories at their own commits on the same side, and the replay here run // against all of them. env := []string{"MESH_REPLAY_CONTROLLER=" + tree} for _, also := range r.With { ref := also.Fix if !fixed { ref = also.Fix + "^1" if also.Before != "" { ref = also.Before } } other, err := os.MkdirTemp("", "prove-"+r.ID+"-"+also.Repository+"-") if err != nil { return "error", err.Error() } defer os.RemoveAll(other) clone := filepath.Join(repos, also.Repository) if out, err := exec.CommandContext(ctx, "git", "-C", clone, "worktree", "add", "--quiet", "--detach", other, ref).CombinedOutput(); err != nil { return "error", string(out) } defer exec.Command("git", "-C", clone, "worktree", "remove", "--force", other).Run() if also.Repository == "mesh-host" { env = append(env, "MESH_REPLAY_HOST="+other) } } cmd := exec.CommandContext(ctx, "go", "test", "-count=1", "-run", "^"+r.Test+"$", r.Package) cmd.Env = append(os.Environ(), env...) out, err := cmd.CombinedOutput() return verdictOf(string(out), err, false) } mode := "-mod=vendor" if _, err := os.Stat(filepath.Join(tree, r.Module, "vendor")); err != nil { mode = "-mod=mod" } cmd := exec.CommandContext(ctx, "go", "test", "-count=1", "-run", "^"+r.Test, r.Package) cmd.Dir = filepath.Join(tree, r.Module) cmd.Env = append(os.Environ(), "GOFLAGS="+mode, "GOPRIVATE=git.novox.be") out, err := cmd.CombinedOutput() return verdictOf(string(out), err, r.Kind == replays.Gate) case replays.Bus: // **The very image the catalogue pinned at the commit**, by its digest: the release a digest is was // not said beside it before the fix, and the digest is what ran. image, err := pinnedBus(tree) if err != nil { return "error", err.Error() } url, stop, err := aBus(ctx, image) if err != nil { return "error", err.Error() } defer stop() return replayHere(ctx, r, "MESH_TEST_NATS="+url) case replays.Resolver: return replayHere(ctx, r, "MESH_REPLAY_CATALOGUE="+tree) } return "error", "no such kind" } func replayHere(ctx context.Context, r replays.Replay, env string) (string, string) { cmd := exec.CommandContext(ctx, "go", "test", "-count=1", "-run", "^"+r.Test, r.Package) cmd.Env = append(os.Environ(), env) out, err := cmd.CombinedOutput() return verdictOf(string(out), err, false) } // verdictOf reads go test's word: a replay that does not build where the check did not exist is absent. func verdictOf(out string, err error, gate bool) (string, string) { switch { case err == nil && strings.Contains(out, "no tests to run"): return "absent", "no test of that name at this commit" case err == nil: return "passes", lastLines(out, 3) case strings.Contains(out, "[build failed]") || strings.Contains(out, "[setup failed]"): if gate { return "absent", "the check did not exist: " + lastLines(out, 3) } return "error", "does not build: " + lastLines(out, 6) case strings.Contains(out, "--- FAIL"): return "fails", failure(out) } return "error", lastLines(out, 6) } // pinnedBus is the bus image the catalogue's nats module is built on at a commit. func pinnedBus(tree string) (string, error) { raw, err := os.ReadFile(filepath.Join(tree, "modules", "nats", "module.json")) if err != nil { return "", err } var m struct { Build struct { On []struct{ Arg, Image string } `json:"on"` } `json:"build"` } if err := json.Unmarshal(raw, &m); err != nil { return "", err } for _, on := range m.Build.On { if on.Arg == "NATS_BASE" && on.Image != "" { return on.Image, nil } } return "", errors.New("the catalogue's nats module names no NATS_BASE image") } // aBus is a server of an image, on loopback, for the length of one replay. func aBus(ctx context.Context, image string) (string, func(), error) { name := fmt.Sprintf("prove-bus-%d", time.Now().UnixNano()) if out, err := exec.CommandContext(ctx, "docker", "run", "-d", "--rm", "--name", name, "-p", "127.0.0.1::4222", image, "-js").CombinedOutput(); err != nil { return "", nil, fmt.Errorf("%s: %v %s", image, err, out) } stop := func() { _ = exec.Command("docker", "rm", "-f", name).Run() } out, err := exec.CommandContext(ctx, "docker", "port", name, "4222/tcp").Output() if err != nil { stop() return "", nil, err } address := strings.TrimSpace(strings.Split(string(out), "\n")[0]) for i := 0; i < 50; i++ { if c, err := net.Dial("tcp", address); err == nil { c.Close() break } time.Sleep(200 * time.Millisecond) } return "nats://" + address, stop, nil } func failure(out string) string { for _, line := range strings.Split(out, "\n") { if strings.Contains(line, "_test.go:") { return strings.TrimSpace(line) } } return lastLines(out, 3) } func lastLines(s string, n int) string { lines := strings.Split(strings.TrimSpace(s), "\n") if len(lines) > n { lines = lines[len(lines)-n:] } return strings.Join(lines, " | ") } func firstLine(s string) string { line, _, _ := strings.Cut(s, "\n") if len(line) > 300 { line = line[:300] + "…" } return line } func contains(list []string, s string) bool { for _, x := range list { if x == s { return true } } return false }