jochen
3abb07f1e2
A module whose only fault was a secret the operator had not given yet was stated unhealthy, failed its first-node gate and was put back. The engine now reads the tool's answer through mesh-sdk go/health and states such a resource waiting, with what it waits for, on one look; a malformed wait stays unhealthy.
205 lines
8.8 KiB
Go
205 lines
8.8 KiB
Go
package liveness
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"os/exec"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-host/internal/declaration"
|
|
"github.com/novox/mesh-host/internal/link"
|
|
)
|
|
|
|
// **The crash loop, on a real runtime** — the engine's half of mesh-lab's replay R-crashloop (novox/hq
|
|
// ADR 0240, "how it is checked", rule 1: "a mesh-lab replay of the crash loop (a container whose program
|
|
// exits at start) fails its gate within the bound").
|
|
//
|
|
// A container whose program exits at once, started as the apply starts every container — detached,
|
|
// restarted by the runtime unless stopped, labelled with its resource — is judged by this engine through
|
|
// the runtime's own command line, and said unhealthy, restarting, inside the gate's bound. What it said is
|
|
// written to MESH_REPLAY_STATEMENT, for the controller's half to judge its gate from.
|
|
//
|
|
// Run by the lab (`go run ./replays/cmd/prove R-crashloop`), which starts the container and names it in
|
|
// MESH_REPLAY_CONTAINER; or by hand with MESH_REPLAY_RUNTIME=1, starting its own. Skipped otherwise: the
|
|
// suite raises no container unasked. The grace is shortened to seconds, which changes the clock and not
|
|
// the rule.
|
|
func TestReplayCrashLoopIsSaidUnhealthy(t *testing.T) {
|
|
name := os.Getenv("MESH_REPLAY_CONTAINER")
|
|
if name == "" && os.Getenv("MESH_REPLAY_RUNTIME") != "1" {
|
|
t.Skip("no MESH_REPLAY_CONTAINER, and MESH_REPLAY_RUNTIME is not 1: this replay raises a container")
|
|
}
|
|
run := func(ctx context.Context, cmd string, args ...string) (string, error) {
|
|
c := exec.CommandContext(ctx, cmd, args...)
|
|
out, err := c.Output()
|
|
var exit *exec.ExitError
|
|
if errors.As(err, &exit) {
|
|
return string(out), fmt.Errorf("%s exited %d: %s", cmd, exit.ExitCode(), strings.TrimSpace(string(exit.Stderr)))
|
|
}
|
|
return string(out), err
|
|
}
|
|
if _, err := run(t.Context(), "docker", "version", "--format", "{{.Server.Version}}"); err != nil {
|
|
t.Skipf("no container runtime answers here: %v", err)
|
|
}
|
|
if name == "" {
|
|
name = fmt.Sprintf("mesh-replay-crashloop-%d", time.Now().UnixNano())
|
|
if out, err := run(t.Context(), "docker", "run", "--detach", "--name", name, "--restart", "unless-stopped",
|
|
"--label", "mesh-host.id=app.server", "--label", "mesh.replay=1", "alpine:3.20", "sh", "-c", "exit 3"); err != nil {
|
|
t.Fatalf("starting the crash loop: %v %s", err, out)
|
|
}
|
|
t.Cleanup(func() { _, _ = run(context.Background(), "docker", "rm", "-f", name) })
|
|
}
|
|
|
|
j, err := Open("", &Exec{Run: run})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
j.Grace = 3 * time.Second
|
|
j.Set([]Resource{{Module: "app", ID: "app.server", Kind: KindContainer, Target: name}})
|
|
began := time.Now()
|
|
deadline := began.Add(2 * time.Minute)
|
|
var st Statement
|
|
for time.Now().Before(deadline) {
|
|
st, _ = j.Look(t.Context())
|
|
if r := st.Resources[0]; r.State == Unhealthy && r.Restarts >= 2 {
|
|
break
|
|
}
|
|
time.Sleep(time.Second)
|
|
}
|
|
s := st.Resources[0]
|
|
if s.State != Unhealthy || s.Restarts < 2 {
|
|
t.Fatalf("a container whose program exits at start was not said unhealthy, with its restarts counted, in two minutes: %+v", s)
|
|
}
|
|
t.Logf("said %s (%s), %d restarts counted, %s after the judging began", s.State, s.Reason, s.Restarts,
|
|
time.Since(began).Round(time.Second))
|
|
|
|
if out := os.Getenv("MESH_REPLAY_STATEMENT"); out != "" {
|
|
h := link.Health{Contract: link.LivenessContract, At: st.At.UTC(), Resources: []link.ResourceHealth{{
|
|
Module: s.Module, Resource: s.ID, Kind: s.Kind, Target: s.Target, State: s.State, Reason: s.Reason,
|
|
Since: s.Since.UTC(), Streak: s.Streak, Restarts: s.Restarts}}}
|
|
raw, err := json.Marshal(h)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(out, raw, 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// SilentWebImage and SilentWebProgram are the silent web application: a web server whose application
|
|
// never answers — it accepts every request and holds it, as the application waiting on a database it could
|
|
// not reach did. The lab raises the same (mesh-lab replays/silentweb_test.go).
|
|
const (
|
|
SilentWebImage = "busybox:1.36"
|
|
SilentWebProgram = "mkdir -p /www/cgi-bin && printf '#!/bin/sh\\nsleep 3600\\n' > /www/cgi-bin/app && " +
|
|
"chmod +x /www/cgi-bin/app && exec httpd -f -p 8080 -h /www"
|
|
)
|
|
|
|
// **R145, the engine's half — a web application that accepts TCP and answers nothing is said unhealthy by
|
|
// its HTTP check within two looks** (novox/hq ADR 0240 Phase B, issue 145). The application's port was
|
|
// open and its program ran while every request hung, for eleven hours. Liveness says it alive and a TCP
|
|
// check says it reachable; only the HTTP check its module declares sees it.
|
|
//
|
|
// A container that accepts every connection on its port and never answers is raised here as the
|
|
// node-engine raises a module's, its port published on the machine; the judge looks at it with an HTTP check
|
|
// of two looks and a TCP check beside it. MESH_REPLAY_STATEMENT, when set, is where the statement is written
|
|
// for the controller's half (mesh-controller TestReplaySilentWebAppIsRaisedWithinTwoLooks). The container is
|
|
// removed after, whatever happened.
|
|
func TestReplaySilentWebAppIsSaidUnhealthy(t *testing.T) {
|
|
if os.Getenv("MESH_REPLAY_RUNTIME") != "1" && os.Getenv("MESH_REPLAY_CONTAINER") == "" {
|
|
t.Skip("no MESH_REPLAY_CONTAINER, and MESH_REPLAY_RUNTIME is not 1: this replay raises a container")
|
|
}
|
|
run := func(ctx context.Context, cmd string, args ...string) (string, error) {
|
|
c := exec.CommandContext(ctx, cmd, args...)
|
|
out, err := c.Output()
|
|
var exit *exec.ExitError
|
|
if errors.As(err, &exit) {
|
|
return string(out), fmt.Errorf("%s exited %d: %s", cmd, exit.ExitCode(), strings.TrimSpace(string(exit.Stderr)))
|
|
}
|
|
return string(out), err
|
|
}
|
|
if _, err := run(t.Context(), "docker", "version", "--format", "{{.Server.Version}}"); err != nil {
|
|
t.Skipf("no container runtime answers here: %v", err)
|
|
}
|
|
// The container is raised by the lab (MESH_REPLAY_CONTAINER, reached at MESH_REPLAY_ADDRESS on its own
|
|
// port), or here, published on this machine's loopback.
|
|
name, host, published := os.Getenv("MESH_REPLAY_CONTAINER"), os.Getenv("MESH_REPLAY_ADDRESS"), 8080
|
|
if name == "" {
|
|
name = fmt.Sprintf("mesh-replay-silent-web-%d", time.Now().UnixNano())
|
|
if out, err := run(t.Context(), "docker", "run", "--detach", "--name", name, "--restart", "unless-stopped",
|
|
"--publish", "127.0.0.1::8080", "--label", "mesh-host.id=app.server", "--label", "mesh.replay=1", SilentWebImage,
|
|
"sh", "-c", SilentWebProgram); err != nil {
|
|
t.Fatalf("starting the silent web application: %v %s", err, out)
|
|
}
|
|
t.Cleanup(func() { _, _ = run(context.Background(), "docker", "rm", "-f", name) })
|
|
said, err := run(t.Context(), "docker", "port", name, "8080/tcp")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_, port, _ := strings.Cut(strings.TrimSpace(strings.Split(said, "\n")[0]), "127.0.0.1:")
|
|
if published, err = strconv.Atoi(port); err != nil {
|
|
t.Fatalf("the port it was published on: %q", said)
|
|
}
|
|
host = "127.0.0.1"
|
|
}
|
|
|
|
timing := func(h *declaration.Health) *declaration.Health {
|
|
h.Interval, h.Timeout, h.Looks, h.Grace = "10s", "2s", 2, "0s"
|
|
return h
|
|
}
|
|
j, err := Open("", &Exec{Run: run})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
j.Set([]Resource{
|
|
{Module: "app", ID: "app.server", Kind: KindContainer, Target: name,
|
|
Check: timing(&declaration.Health{Kind: declaration.HealthHTTP, Endpoint: "web", Port: published, Path: "/cgi-bin/app"})},
|
|
})
|
|
j.Probes = &Probes{Host: host}
|
|
tcp := j.Probes
|
|
began := time.Now()
|
|
var st Statement
|
|
looks := 0
|
|
for time.Since(began) < 2*time.Minute {
|
|
for _, d := range j.due() {
|
|
ok, why, waits := j.probes().Look(t.Context(), d.module, d.check)
|
|
j.Record(d.id, d.started, ok, why, waits...)
|
|
looks++
|
|
}
|
|
st, _ = j.Look(t.Context())
|
|
if st.Resources[0].State == Unhealthy {
|
|
break
|
|
}
|
|
time.Sleep(time.Second)
|
|
}
|
|
s := st.Resources[0]
|
|
if s.State != Unhealthy || looks > 2 {
|
|
t.Fatalf("a web application answering nothing was not said unhealthy within two looks (%d looks): %+v", looks, s)
|
|
}
|
|
// And a TCP check would have said it reachable: the port is open.
|
|
if ok, why, _ := tcp.Look(t.Context(), "app", timing(&declaration.Health{Kind: declaration.HealthTCP, Port: published})); !ok {
|
|
t.Fatalf("a tcp check could not connect to the silent application: %s", why)
|
|
}
|
|
t.Logf("said %s (%s) after %d looks, %s after the judging began; a TCP check connects", s.State, s.Reason, looks,
|
|
time.Since(began).Round(time.Second))
|
|
|
|
if out := os.Getenv("MESH_REPLAY_STATEMENT"); out != "" {
|
|
h := link.Health{Contract: link.ReadinessContract, At: st.At.UTC(), Resources: []link.ResourceHealth{{
|
|
Module: s.Module, Resource: s.ID, Kind: s.Kind, Target: s.Target, State: s.State, Reason: s.Reason,
|
|
Since: s.Since.UTC(), Streak: s.Streak, Restarts: s.Restarts, Check: s.CheckOf(), Needs: s.NeedsOf()}}}
|
|
raw, err := json.Marshal(h)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(out, raw, 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
}
|