package liveness import ( "context" "encoding/json" "errors" "fmt" "os" "os/exec" "strconv" "strings" "testing" "time" "github.com/novox/mesh-host/internal/declaration" "github.com/novox/mesh-host/internal/link" ) // **The crash loop, on a real runtime** — the engine's half of mesh-lab's replay R-crashloop (novox/hq // ADR 0240, "how it is checked", rule 1: "a mesh-lab replay of the crash loop (a container whose program // exits at start) fails its gate within the bound"). // // A container whose program exits at once, started as the apply starts every container — detached, // restarted by the runtime unless stopped, labelled with its resource — is judged by this engine through // the runtime's own command line, and said unhealthy, restarting, inside the gate's bound. What it said is // written to MESH_REPLAY_STATEMENT, for the controller's half to judge its gate from. // // Run by the lab (`go run ./replays/cmd/prove R-crashloop`), which starts the container and names it in // MESH_REPLAY_CONTAINER; or by hand with MESH_REPLAY_RUNTIME=1, starting its own. Skipped otherwise: the // suite raises no container unasked. The grace is shortened to seconds, which changes the clock and not // the rule. func TestReplayCrashLoopIsSaidUnhealthy(t *testing.T) { name := os.Getenv("MESH_REPLAY_CONTAINER") if name == "" && os.Getenv("MESH_REPLAY_RUNTIME") != "1" { t.Skip("no MESH_REPLAY_CONTAINER, and MESH_REPLAY_RUNTIME is not 1: this replay raises a container") } run := func(ctx context.Context, cmd string, args ...string) (string, error) { c := exec.CommandContext(ctx, cmd, args...) out, err := c.Output() var exit *exec.ExitError if errors.As(err, &exit) { return string(out), fmt.Errorf("%s exited %d: %s", cmd, exit.ExitCode(), strings.TrimSpace(string(exit.Stderr))) } return string(out), err } if _, err := run(t.Context(), "docker", "version", "--format", "{{.Server.Version}}"); err != nil { t.Skipf("no container runtime answers here: %v", err) } if name == "" { name = fmt.Sprintf("mesh-replay-crashloop-%d", time.Now().UnixNano()) if out, err := run(t.Context(), "docker", "run", "--detach", "--name", name, "--restart", "unless-stopped", "--label", "mesh-host.id=app.server", "--label", "mesh.replay=1", "alpine:3.20", "sh", "-c", "exit 3"); err != nil { t.Fatalf("starting the crash loop: %v %s", err, out) } t.Cleanup(func() { _, _ = run(context.Background(), "docker", "rm", "-f", name) }) } j, err := Open("", &Exec{Run: run}) if err != nil { t.Fatal(err) } j.Grace = 3 * time.Second j.Set([]Resource{{Module: "app", ID: "app.server", Kind: KindContainer, Target: name}}) began := time.Now() deadline := began.Add(2 * time.Minute) var st Statement for time.Now().Before(deadline) { st, _ = j.Look(t.Context()) if r := st.Resources[0]; r.State == Unhealthy && r.Restarts >= 2 { break } time.Sleep(time.Second) } s := st.Resources[0] if s.State != Unhealthy || s.Restarts < 2 { t.Fatalf("a container whose program exits at start was not said unhealthy, with its restarts counted, in two minutes: %+v", s) } t.Logf("said %s (%s), %d restarts counted, %s after the judging began", s.State, s.Reason, s.Restarts, time.Since(began).Round(time.Second)) if out := os.Getenv("MESH_REPLAY_STATEMENT"); out != "" { h := link.Health{Contract: link.LivenessContract, At: st.At.UTC(), Resources: []link.ResourceHealth{{ Module: s.Module, Resource: s.ID, Kind: s.Kind, Target: s.Target, State: s.State, Reason: s.Reason, Since: s.Since.UTC(), Streak: s.Streak, Restarts: s.Restarts}}} raw, err := json.Marshal(h) if err != nil { t.Fatal(err) } if err := os.WriteFile(out, raw, 0o644); err != nil { t.Fatal(err) } } } // SilentWebImage and SilentWebProgram are the silent web application: a web server whose application // never answers — it accepts every request and holds it, as the application waiting on a database it could // not reach did. The lab raises the same (mesh-lab replays/silentweb_test.go). const ( SilentWebImage = "busybox:1.36" SilentWebProgram = "mkdir -p /www/cgi-bin && printf '#!/bin/sh\\nsleep 3600\\n' > /www/cgi-bin/app && " + "chmod +x /www/cgi-bin/app && exec httpd -f -p 8080 -h /www" ) // **R145, the engine's half — a web application that accepts TCP and answers nothing is said unhealthy by // its HTTP check within two looks** (novox/hq ADR 0240 Phase B, issue 145). The application's port was // open and its program ran while every request hung, for eleven hours. Liveness says it alive and a TCP // check says it reachable; only the HTTP check its module declares sees it. // // A container that accepts every connection on its port and never answers is raised here as the // node-engine raises a module's, its port published on the machine; the judge looks at it with an HTTP check // of two looks and a TCP check beside it. MESH_REPLAY_STATEMENT, when set, is where the statement is written // for the controller's half (mesh-controller TestReplaySilentWebAppIsRaisedWithinTwoLooks). The container is // removed after, whatever happened. func TestReplaySilentWebAppIsSaidUnhealthy(t *testing.T) { if os.Getenv("MESH_REPLAY_RUNTIME") != "1" && os.Getenv("MESH_REPLAY_CONTAINER") == "" { t.Skip("no MESH_REPLAY_CONTAINER, and MESH_REPLAY_RUNTIME is not 1: this replay raises a container") } run := func(ctx context.Context, cmd string, args ...string) (string, error) { c := exec.CommandContext(ctx, cmd, args...) out, err := c.Output() var exit *exec.ExitError if errors.As(err, &exit) { return string(out), fmt.Errorf("%s exited %d: %s", cmd, exit.ExitCode(), strings.TrimSpace(string(exit.Stderr))) } return string(out), err } if _, err := run(t.Context(), "docker", "version", "--format", "{{.Server.Version}}"); err != nil { t.Skipf("no container runtime answers here: %v", err) } // The container is raised by the lab (MESH_REPLAY_CONTAINER, reached at MESH_REPLAY_ADDRESS on its own // port), or here, published on this machine's loopback. name, host, published := os.Getenv("MESH_REPLAY_CONTAINER"), os.Getenv("MESH_REPLAY_ADDRESS"), 8080 if name == "" { name = fmt.Sprintf("mesh-replay-silent-web-%d", time.Now().UnixNano()) if out, err := run(t.Context(), "docker", "run", "--detach", "--name", name, "--restart", "unless-stopped", "--publish", "127.0.0.1::8080", "--label", "mesh-host.id=app.server", "--label", "mesh.replay=1", SilentWebImage, "sh", "-c", SilentWebProgram); err != nil { t.Fatalf("starting the silent web application: %v %s", err, out) } t.Cleanup(func() { _, _ = run(context.Background(), "docker", "rm", "-f", name) }) said, err := run(t.Context(), "docker", "port", name, "8080/tcp") if err != nil { t.Fatal(err) } _, port, _ := strings.Cut(strings.TrimSpace(strings.Split(said, "\n")[0]), "127.0.0.1:") if published, err = strconv.Atoi(port); err != nil { t.Fatalf("the port it was published on: %q", said) } host = "127.0.0.1" } timing := func(h *declaration.Health) *declaration.Health { h.Interval, h.Timeout, h.Looks, h.Grace = "10s", "2s", 2, "0s" return h } j, err := Open("", &Exec{Run: run}) if err != nil { t.Fatal(err) } j.Set([]Resource{ {Module: "app", ID: "app.server", Kind: KindContainer, Target: name, Check: timing(&declaration.Health{Kind: declaration.HealthHTTP, Endpoint: "web", Port: published, Path: "/cgi-bin/app"})}, }) j.Probes = &Probes{Host: host} tcp := j.Probes began := time.Now() var st Statement looks := 0 for time.Since(began) < 2*time.Minute { for _, d := range j.due() { ok, why := j.probes().Look(t.Context(), d.module, d.check) j.Record(d.id, d.started, ok, why) looks++ } st, _ = j.Look(t.Context()) if st.Resources[0].State == Unhealthy { break } time.Sleep(time.Second) } s := st.Resources[0] if s.State != Unhealthy || looks > 2 { t.Fatalf("a web application answering nothing was not said unhealthy within two looks (%d looks): %+v", looks, s) } // And a TCP check would have said it reachable: the port is open. if ok, why := tcp.Look(t.Context(), "app", timing(&declaration.Health{Kind: declaration.HealthTCP, Port: published})); !ok { t.Fatalf("a tcp check could not connect to the silent application: %s", why) } t.Logf("said %s (%s) after %d looks, %s after the judging began; a TCP check connects", s.State, s.Reason, looks, time.Since(began).Round(time.Second)) if out := os.Getenv("MESH_REPLAY_STATEMENT"); out != "" { h := link.Health{Contract: link.ReadinessContract, At: st.At.UTC(), Resources: []link.ResourceHealth{{ Module: s.Module, Resource: s.ID, Kind: s.Kind, Target: s.Target, State: s.State, Reason: s.Reason, Since: s.Since.UTC(), Streak: s.Streak, Restarts: s.Restarts, Check: s.CheckOf(), Needs: s.NeedsOf()}}} raw, err := json.Marshal(h) if err != nil { t.Fatal(err) } if err := os.WriteFile(out, raw, 0o644); err != nil { t.Fatal(err) } } }