package replays import ( "context" "fmt" "os" "os/exec" "path/filepath" "strings" "testing" "time" ) // **R-crashloop — a module's container that crash-loops fails its gate on the first machine** (novox/hq // ADR 0240, "how it is checked", rule 1; to-be 48 Phase A, done when). // // On the home server the agent server restarted about a hundred times in a crash loop while the mesh read // it applied, its tools served and no condition raised; a person found it reading its log for another // reason (issue 268, research 032 §6). The gate judged a module by what the mesh saw from outside. // // The replay is the whole chain, each half at its commit: // // 1. a container whose program exits at start is raised here as the node-engine raises a module's — the // runtime restarting it unless stopped, labelled with its resource; // 2. the node-engine at MESH_REPLAY_HOST judges it through the runtime, as it judges every long-running // resource, and writes what it states (its TestReplayCrashLoopIsSaidUnhealthy) — an engine older than // the judging has nothing to state, and its reports carry no health; // 3. the controller at MESH_REPLAY_CONTROLLER rolls a build of the module out, hears its first machine say // that, and judges the gate (its TestReplayCrashLoopFailsItsGateOnTheFirstMachine). // // The assertion is the controller's: the build fails its gate on the first machine and is put back there. // Needs a container runtime, Go, and MESH_TEST_POSTGRES for the controller's store; the container is // removed after, whatever happened. func TestReplayCrashLoop(t *testing.T) { host := orDefault(os.Getenv("MESH_REPLAY_HOST"), filepath.Join("..", "..", "mesh-host")) controller := orDefault(os.Getenv("MESH_REPLAY_CONTROLLER"), filepath.Join("..", "..", "mesh-controller")) for _, dir := range []string{host, controller} { if _, err := os.Stat(filepath.Join(dir, "go.mod")); err != nil { t.Skipf("no checkout at %s (MESH_REPLAY_HOST and MESH_REPLAY_CONTROLLER name them)", dir) } } if os.Getenv("MESH_TEST_POSTGRES") == "" { t.Skip("no MESH_TEST_POSTGRES: the controller's half judges its gate against a store") } docker, ok := DockerFromEnv() if !ok { t.Skip("no container runtime: the crash loop is a container") } ctx, cancel := context.WithTimeout(t.Context(), 10*time.Minute) defer cancel() // 1. The crash loop, as the node-engine raises a module's container. const image = "alpine:3.20" if err := docker.Pull(ctx, image); err != nil { t.Fatal(err) } name := fmt.Sprintf("mesh-replay-crashloop-%d", time.Now().UnixNano()) id, _, err := docker.Start(ctx, Run{Image: image, Cmd: []string{"sh", "-c", "echo starting; exit 3"}, Name: name, Restart: "unless-stopped", Labels: map[string]string{"mesh-host.id": "app.server"}}) if err != nil { t.Fatal(err) } defer docker.Remove(context.Background(), id) // 2. The node-engine at its commit judges it, and says what it states. statement := filepath.Join(t.TempDir(), "statement.json") engine := exec.CommandContext(ctx, "go", "test", "-count=1", "-run", "^TestReplayCrashLoopIsSaidUnhealthy$", "./internal/liveness/") engine.Dir = host engine.Env = append(os.Environ(), "MESH_REPLAY_CONTAINER="+name, "MESH_REPLAY_STATEMENT="+statement, "GOFLAGS=") out, err := engine.CombinedOutput() switch { case err == nil && !strings.Contains(string(out), "no tests to run"): t.Logf("the node-engine said: %s", firstLineOf(statementOf(statement))) case judgesNothing(string(out)): // An engine older than the judging: its reports carry no health. t.Logf("the node-engine at this commit judges nothing it runs") if err := os.WriteFile(statement, []byte("null"), 0o644); err != nil { t.Fatal(err) } default: t.Fatalf("the node-engine did not say the crash loop unhealthy: %v\n%s", err, lastOf(string(out), 12)) } // 3. The controller at its commit judges the gate from what the engine said. gate := exec.CommandContext(ctx, "go", "test", "-count=1", "-run", "^TestReplayCrashLoopFailsItsGateOnTheFirstMachine$", "./cmd/mesh-controller/") gate.Dir = controller gate.Env = append(os.Environ(), "MESH_REPLAY_STATEMENT="+statement, "GOFLAGS=-mod=vendor", "GOPROXY=off") out, err = gate.CombinedOutput() if err != nil { t.Fatalf("the gate did not fail the crash loop on its first machine: %v\n%s", err, lastOf(string(out), 12)) } if strings.Contains(string(out), "no tests to run") { t.Fatal("the controller at this commit has no half of the replay to run") } } // judgesNothing is go test's word for a package that is not there: an engine older than the judging. func judgesNothing(out string) bool { return strings.Contains(out, "no tests to run") || strings.Contains(out, "directory not found") || strings.Contains(out, "no Go files") || strings.Contains(out, "cannot find package") || strings.Contains(out, "matched no packages") || strings.Contains(out, "is not in std") } func statementOf(path string) string { raw, err := os.ReadFile(path) if err != nil { return "(nothing written: " + err.Error() + ")" } return string(raw) } func orDefault(s, fallback string) string { if s == "" { return fallback } return s } func firstLineOf(s string) string { line, _, _ := strings.Cut(strings.TrimSpace(s), "\n") if len(line) > 400 { line = line[:400] + "…" } return line } func lastOf(s string, n int) string { lines := strings.Split(strings.TrimSpace(s), "\n") if len(lines) > n { lines = lines[len(lines)-n:] } return strings.Join(lines, "\n") }