jochen
bdd44154cc
Liveness alone could not see a web application whose port was open and whose program ran while every request hung for eleven hours (issue 145). A resource now carries the `health` its module declared: the engine makes http and tcp looks itself from the machine to the endpoint's published port, reads a unit's readiness from the show it already makes, hands an exec command or the image's own check to the runtime as the container's check with the declared timing and reads its state from the inspect it already makes, and asks a module's tool on its own node tools. Starting until the check passed, unhealthy once its looks after the grace fail the declared number of times; never more looks than the measured budget; nothing restarted. The statement says contract 2, which tells the controller this engine may be sent the field.
205 lines
8.8 KiB
Go
205 lines
8.8 KiB
Go
package liveness
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"os/exec"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-host/internal/declaration"
|
|
"github.com/novox/mesh-host/internal/link"
|
|
)
|
|
|
|
// **The crash loop, on a real runtime** — the engine's half of mesh-lab's replay R-crashloop (novox/hq
|
|
// ADR 0240, "how it is checked", rule 1: "a mesh-lab replay of the crash loop (a container whose program
|
|
// exits at start) fails its gate within the bound").
|
|
//
|
|
// A container whose program exits at once, started as the apply starts every container — detached,
|
|
// restarted by the runtime unless stopped, labelled with its resource — is judged by this engine through
|
|
// the runtime's own command line, and said unhealthy, restarting, inside the gate's bound. What it said is
|
|
// written to MESH_REPLAY_STATEMENT, for the controller's half to judge its gate from.
|
|
//
|
|
// Run by the lab (`go run ./replays/cmd/prove R-crashloop`), which starts the container and names it in
|
|
// MESH_REPLAY_CONTAINER; or by hand with MESH_REPLAY_RUNTIME=1, starting its own. Skipped otherwise: the
|
|
// suite raises no container unasked. The grace is shortened to seconds, which changes the clock and not
|
|
// the rule.
|
|
func TestReplayCrashLoopIsSaidUnhealthy(t *testing.T) {
|
|
name := os.Getenv("MESH_REPLAY_CONTAINER")
|
|
if name == "" && os.Getenv("MESH_REPLAY_RUNTIME") != "1" {
|
|
t.Skip("no MESH_REPLAY_CONTAINER, and MESH_REPLAY_RUNTIME is not 1: this replay raises a container")
|
|
}
|
|
run := func(ctx context.Context, cmd string, args ...string) (string, error) {
|
|
c := exec.CommandContext(ctx, cmd, args...)
|
|
out, err := c.Output()
|
|
var exit *exec.ExitError
|
|
if errors.As(err, &exit) {
|
|
return string(out), fmt.Errorf("%s exited %d: %s", cmd, exit.ExitCode(), strings.TrimSpace(string(exit.Stderr)))
|
|
}
|
|
return string(out), err
|
|
}
|
|
if _, err := run(t.Context(), "docker", "version", "--format", "{{.Server.Version}}"); err != nil {
|
|
t.Skipf("no container runtime answers here: %v", err)
|
|
}
|
|
if name == "" {
|
|
name = fmt.Sprintf("mesh-replay-crashloop-%d", time.Now().UnixNano())
|
|
if out, err := run(t.Context(), "docker", "run", "--detach", "--name", name, "--restart", "unless-stopped",
|
|
"--label", "mesh-host.id=app.server", "--label", "mesh.replay=1", "alpine:3.20", "sh", "-c", "exit 3"); err != nil {
|
|
t.Fatalf("starting the crash loop: %v %s", err, out)
|
|
}
|
|
t.Cleanup(func() { _, _ = run(context.Background(), "docker", "rm", "-f", name) })
|
|
}
|
|
|
|
j, err := Open("", &Exec{Run: run})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
j.Grace = 3 * time.Second
|
|
j.Set([]Resource{{Module: "app", ID: "app.server", Kind: KindContainer, Target: name}})
|
|
began := time.Now()
|
|
deadline := began.Add(2 * time.Minute)
|
|
var st Statement
|
|
for time.Now().Before(deadline) {
|
|
st, _ = j.Look(t.Context())
|
|
if r := st.Resources[0]; r.State == Unhealthy && r.Restarts >= 2 {
|
|
break
|
|
}
|
|
time.Sleep(time.Second)
|
|
}
|
|
s := st.Resources[0]
|
|
if s.State != Unhealthy || s.Restarts < 2 {
|
|
t.Fatalf("a container whose program exits at start was not said unhealthy, with its restarts counted, in two minutes: %+v", s)
|
|
}
|
|
t.Logf("said %s (%s), %d restarts counted, %s after the judging began", s.State, s.Reason, s.Restarts,
|
|
time.Since(began).Round(time.Second))
|
|
|
|
if out := os.Getenv("MESH_REPLAY_STATEMENT"); out != "" {
|
|
h := link.Health{Contract: link.LivenessContract, At: st.At.UTC(), Resources: []link.ResourceHealth{{
|
|
Module: s.Module, Resource: s.ID, Kind: s.Kind, Target: s.Target, State: s.State, Reason: s.Reason,
|
|
Since: s.Since.UTC(), Streak: s.Streak, Restarts: s.Restarts}}}
|
|
raw, err := json.Marshal(h)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(out, raw, 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// SilentWebImage and SilentWebProgram are the silent web application: a web server whose application
|
|
// never answers — it accepts every request and holds it, as the application waiting on a database it could
|
|
// not reach did. The lab raises the same (mesh-lab replays/silentweb_test.go).
|
|
const (
|
|
SilentWebImage = "busybox:1.36"
|
|
SilentWebProgram = "mkdir -p /www/cgi-bin && printf '#!/bin/sh\\nsleep 3600\\n' > /www/cgi-bin/app && " +
|
|
"chmod +x /www/cgi-bin/app && exec httpd -f -p 8080 -h /www"
|
|
)
|
|
|
|
// **R145, the engine's half — a web application that accepts TCP and answers nothing is said unhealthy by
|
|
// its HTTP check within two looks** (novox/hq ADR 0240 Phase B, issue 145). The application's port was
|
|
// open and its program ran while every request hung, for eleven hours. Liveness says it alive and a TCP
|
|
// check says it reachable; only the HTTP check its module declares sees it.
|
|
//
|
|
// A container that accepts every connection on its port and never answers is raised here as the
|
|
// node-engine raises a module's, its port published on the machine; the judge looks at it with an HTTP check
|
|
// of two looks and a TCP check beside it. MESH_REPLAY_STATEMENT, when set, is where the statement is written
|
|
// for the controller's half (mesh-controller TestReplaySilentWebAppIsRaisedWithinTwoLooks). The container is
|
|
// removed after, whatever happened.
|
|
func TestReplaySilentWebAppIsSaidUnhealthy(t *testing.T) {
|
|
if os.Getenv("MESH_REPLAY_RUNTIME") != "1" && os.Getenv("MESH_REPLAY_CONTAINER") == "" {
|
|
t.Skip("no MESH_REPLAY_CONTAINER, and MESH_REPLAY_RUNTIME is not 1: this replay raises a container")
|
|
}
|
|
run := func(ctx context.Context, cmd string, args ...string) (string, error) {
|
|
c := exec.CommandContext(ctx, cmd, args...)
|
|
out, err := c.Output()
|
|
var exit *exec.ExitError
|
|
if errors.As(err, &exit) {
|
|
return string(out), fmt.Errorf("%s exited %d: %s", cmd, exit.ExitCode(), strings.TrimSpace(string(exit.Stderr)))
|
|
}
|
|
return string(out), err
|
|
}
|
|
if _, err := run(t.Context(), "docker", "version", "--format", "{{.Server.Version}}"); err != nil {
|
|
t.Skipf("no container runtime answers here: %v", err)
|
|
}
|
|
// The container is raised by the lab (MESH_REPLAY_CONTAINER, reached at MESH_REPLAY_ADDRESS on its own
|
|
// port), or here, published on this machine's loopback.
|
|
name, host, published := os.Getenv("MESH_REPLAY_CONTAINER"), os.Getenv("MESH_REPLAY_ADDRESS"), 8080
|
|
if name == "" {
|
|
name = fmt.Sprintf("mesh-replay-silent-web-%d", time.Now().UnixNano())
|
|
if out, err := run(t.Context(), "docker", "run", "--detach", "--name", name, "--restart", "unless-stopped",
|
|
"--publish", "127.0.0.1::8080", "--label", "mesh-host.id=app.server", "--label", "mesh.replay=1", SilentWebImage,
|
|
"sh", "-c", SilentWebProgram); err != nil {
|
|
t.Fatalf("starting the silent web application: %v %s", err, out)
|
|
}
|
|
t.Cleanup(func() { _, _ = run(context.Background(), "docker", "rm", "-f", name) })
|
|
said, err := run(t.Context(), "docker", "port", name, "8080/tcp")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_, port, _ := strings.Cut(strings.TrimSpace(strings.Split(said, "\n")[0]), "127.0.0.1:")
|
|
if published, err = strconv.Atoi(port); err != nil {
|
|
t.Fatalf("the port it was published on: %q", said)
|
|
}
|
|
host = "127.0.0.1"
|
|
}
|
|
|
|
timing := func(h *declaration.Health) *declaration.Health {
|
|
h.Interval, h.Timeout, h.Looks, h.Grace = "10s", "2s", 2, "0s"
|
|
return h
|
|
}
|
|
j, err := Open("", &Exec{Run: run})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
j.Set([]Resource{
|
|
{Module: "app", ID: "app.server", Kind: KindContainer, Target: name,
|
|
Check: timing(&declaration.Health{Kind: declaration.HealthHTTP, Endpoint: "web", Port: published, Path: "/cgi-bin/app"})},
|
|
})
|
|
j.Probes = &Probes{Host: host}
|
|
tcp := j.Probes
|
|
began := time.Now()
|
|
var st Statement
|
|
looks := 0
|
|
for time.Since(began) < 2*time.Minute {
|
|
for _, d := range j.due() {
|
|
ok, why := j.probes().Look(t.Context(), d.module, d.check)
|
|
j.Record(d.id, d.started, ok, why)
|
|
looks++
|
|
}
|
|
st, _ = j.Look(t.Context())
|
|
if st.Resources[0].State == Unhealthy {
|
|
break
|
|
}
|
|
time.Sleep(time.Second)
|
|
}
|
|
s := st.Resources[0]
|
|
if s.State != Unhealthy || looks > 2 {
|
|
t.Fatalf("a web application answering nothing was not said unhealthy within two looks (%d looks): %+v", looks, s)
|
|
}
|
|
// And a TCP check would have said it reachable: the port is open.
|
|
if ok, why := tcp.Look(t.Context(), "app", timing(&declaration.Health{Kind: declaration.HealthTCP, Port: published})); !ok {
|
|
t.Fatalf("a tcp check could not connect to the silent application: %s", why)
|
|
}
|
|
t.Logf("said %s (%s) after %d looks, %s after the judging began; a TCP check connects", s.State, s.Reason, looks,
|
|
time.Since(began).Round(time.Second))
|
|
|
|
if out := os.Getenv("MESH_REPLAY_STATEMENT"); out != "" {
|
|
h := link.Health{Contract: link.ReadinessContract, At: st.At.UTC(), Resources: []link.ResourceHealth{{
|
|
Module: s.Module, Resource: s.ID, Kind: s.Kind, Target: s.Target, State: s.State, Reason: s.Reason,
|
|
Since: s.Since.UTC(), Streak: s.Streak, Restarts: s.Restarts, Check: s.CheckOf(), Needs: s.NeedsOf()}}}
|
|
raw, err := json.Marshal(h)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(out, raw, 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
}
|