mesh-lab/replays/crashloop_test.go

package replays

import (
	"context"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"testing"
	"time"
)

// **R-crashloop — a module's container that crash-loops fails its gate on the first machine** (novox/hq
// ADR 0240, "how it is checked", rule 1; to-be 48 Phase A, done when).
//
// On the home server the agent server restarted about a hundred times in a crash loop while the mesh read
// it applied, its tools served and no condition raised; a person found it reading its log for another
// reason (issue 268, research 032 §6). The gate judged a module by what the mesh saw from outside.
//
// The replay is the whole chain, each half at its commit:
//
//  1. a container whose program exits at start is raised here as the node-engine raises a module's — the
//     runtime restarting it unless stopped, labelled with its resource;
//  2. the node-engine at MESH_REPLAY_HOST judges it through the runtime, as it judges every long-running
//     resource, and writes what it states (its TestReplayCrashLoopIsSaidUnhealthy) — an engine older than
//     the judging has nothing to state, and its reports carry no health;
//  3. the controller at MESH_REPLAY_CONTROLLER rolls a build of the module out, hears its first machine say
//     that, and judges the gate (its TestReplayCrashLoopFailsItsGateOnTheFirstMachine).
//
// The assertion is the controller's: the build fails its gate on the first machine and is put back there.
// Needs a container runtime, Go, and MESH_TEST_POSTGRES for the controller's store; the container is
// removed after, whatever happened.
func TestReplayCrashLoop(t *testing.T) {
	host := orDefault(os.Getenv("MESH_REPLAY_HOST"), filepath.Join("..", "..", "mesh-host"))
	controller := orDefault(os.Getenv("MESH_REPLAY_CONTROLLER"), filepath.Join("..", "..", "mesh-controller"))
	for _, dir := range []string{host, controller} {
		if _, err := os.Stat(filepath.Join(dir, "go.mod")); err != nil {
			t.Skipf("no checkout at %s (MESH_REPLAY_HOST and MESH_REPLAY_CONTROLLER name them)", dir)
		}
	}
	if os.Getenv("MESH_TEST_POSTGRES") == "" {
		t.Skip("no MESH_TEST_POSTGRES: the controller's half judges its gate against a store")
	}
	docker, ok := DockerFromEnv()
	if !ok {
		t.Skip("no container runtime: the crash loop is a container")
	}
	ctx, cancel := context.WithTimeout(t.Context(), 10*time.Minute)
	defer cancel()

	// 1. The crash loop, as the node-engine raises a module's container.
	const image = "alpine:3.20"
	if err := docker.Pull(ctx, image); err != nil {
		t.Fatal(err)
	}
	name := fmt.Sprintf("mesh-replay-crashloop-%d", time.Now().UnixNano())
	id, _, err := docker.Start(ctx, Run{Image: image, Cmd: []string{"sh", "-c", "echo starting; exit 3"}, Name: name,
		Restart: "unless-stopped", Labels: map[string]string{"mesh-host.id": "app.server"}})
	if err != nil {
		t.Fatal(err)
	}
	defer docker.Remove(context.Background(), id)

	// 2. The node-engine at its commit judges it, and says what it states.
	statement := filepath.Join(t.TempDir(), "statement.json")
	engine := exec.CommandContext(ctx, "go", "test", "-count=1", "-run", "^TestReplayCrashLoopIsSaidUnhealthy$",
		"./internal/liveness/")
	engine.Dir = host
	engine.Env = append(os.Environ(), "MESH_REPLAY_CONTAINER="+name, "MESH_REPLAY_STATEMENT="+statement, "GOFLAGS=")
	out, err := engine.CombinedOutput()
	switch {
	case err == nil && !strings.Contains(string(out), "no tests to run"):
		t.Logf("the node-engine said: %s", firstLineOf(statementOf(statement)))
	case judgesNothing(string(out)):
		// An engine older than the judging: its reports carry no health.
		t.Logf("the node-engine at this commit judges nothing it runs")
		if err := os.WriteFile(statement, []byte("null"), 0o644); err != nil {
			t.Fatal(err)
		}
	default:
		t.Fatalf("the node-engine did not say the crash loop unhealthy: %v\n%s", err, lastOf(string(out), 12))
	}

	// 3. The controller at its commit judges the gate from what the engine said.
	gate := exec.CommandContext(ctx, "go", "test", "-count=1", "-run", "^TestReplayCrashLoopFailsItsGateOnTheFirstMachine$",
		"./cmd/mesh-controller/")
	gate.Dir = controller
	gate.Env = append(os.Environ(), "MESH_REPLAY_STATEMENT="+statement, "GOFLAGS=-mod=vendor", "GOPROXY=off")
	out, err = gate.CombinedOutput()
	if err != nil {
		t.Fatalf("the gate did not fail the crash loop on its first machine: %v\n%s", err, lastOf(string(out), 12))
	}
	if strings.Contains(string(out), "no tests to run") {
		t.Fatal("the controller at this commit has no half of the replay to run")
	}
}

// judgesNothing is go test's word for a package that is not there: an engine older than the judging.
func judgesNothing(out string) bool {
	return strings.Contains(out, "no tests to run") || strings.Contains(out, "directory not found") ||
		strings.Contains(out, "no Go files") || strings.Contains(out, "cannot find package") ||
		strings.Contains(out, "matched no packages") || strings.Contains(out, "is not in std")
}

func statementOf(path string) string {
	raw, err := os.ReadFile(path)
	if err != nil {
		return "(nothing written: " + err.Error() + ")"
	}
	return string(raw)
}

func orDefault(s, fallback string) string {
	if s == "" {
		return fallback
	}
	return s
}

func firstLineOf(s string) string {
	line, _, _ := strings.Cut(strings.TrimSpace(s), "\n")
	if len(line) > 400 {
		line = line[:400] + "…"
	}
	return line
}

func lastOf(s string, n int) string {
	lines := strings.Split(strings.TrimSpace(s), "\n")
	if len(lines) > n {
		lines = lines[len(lines)-n:]
	}
	return strings.Join(lines, "\n")
}