jochen
bdd44154cc
Liveness alone could not see a web application whose port was open and whose program ran while every request hung for eleven hours (issue 145). A resource now carries the `health` its module declared: the engine makes http and tcp looks itself from the machine to the endpoint's published port, reads a unit's readiness from the show it already makes, hands an exec command or the image's own check to the runtime as the container's check with the declared timing and reads its state from the inspect it already makes, and asks a module's tool on its own node tools. Starting until the check passed, unhealthy once its looks after the grace fail the declared number of times; never more looks than the measured budget; nothing restarted. The statement says contract 2, which tells the controller this engine may be sent the field.
232 lines
9.1 KiB
Go
232 lines
9.1 KiB
Go
package liveness
|
|
|
|
import (
|
|
"context"
|
|
"net"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-host/internal/declaration"
|
|
)
|
|
|
|
// Readiness, declared (novox/hq ADR 0240 rules 2 and 3, Phase B): starting until its check passed, healthy
|
|
// once it has, unhealthy once its failing looks after the grace reach the declared number; an http check
|
|
// dials the endpoint's current port after a port change; the runtime's check is read, never run by the
|
|
// engine; and never more looks than the budget.
|
|
|
|
func httpCheck(port int) *declaration.Health {
|
|
return &declaration.Health{Kind: declaration.HealthHTTP, Endpoint: "web", Port: port, Path: "/healthz",
|
|
Interval: "10s", Timeout: "2s", Looks: 2, Grace: "30s"}
|
|
}
|
|
|
|
func portOf(t *testing.T, url string) int {
|
|
t.Helper()
|
|
_, p, err := net.SplitHostPort(strings.TrimPrefix(url, "http://"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
n, _ := strconv.Atoi(p)
|
|
return n
|
|
}
|
|
|
|
// lookOnce makes the engine's own due looks at once and waits for them, as Probe would on its tick.
|
|
func lookOnce(t *testing.T, j *Judge) {
|
|
t.Helper()
|
|
for _, d := range j.due() {
|
|
ok, why := j.probes().Look(t.Context(), d.module, d.check)
|
|
j.Record(d.id, d.started, ok, why)
|
|
}
|
|
}
|
|
|
|
func TestAnHTTPCheckIsStartingUntilItPassesAndUnhealthyAfterItsLooksFail(t *testing.T) {
|
|
answering := true
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if !answering || r.URL.Path != "/healthz" {
|
|
w.WriteHeader(http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
w.Write([]byte("ok"))
|
|
}))
|
|
defer srv.Close()
|
|
c := &clock{now: t0}
|
|
rt := &fakeRuntime{containers: map[string]Observed{"web": running("c1", 0, t0.Format(time.RFC3339Nano))}}
|
|
j := aJudge(t, rt, c, filepath.Join(t.TempDir(), FileName))
|
|
web := Resource{Module: "app", ID: "app.web", Kind: KindContainer, Target: "web", Check: httpCheck(portOf(t, srv.URL))}
|
|
j.Set([]Resource{web})
|
|
|
|
// In its grace, before any look: starting.
|
|
st, _ := j.Look(t.Context())
|
|
if s := stateOf(t, st, "app.web"); s.State != Starting || s.CheckOf() != "http" {
|
|
t.Fatalf("before any look: %+v", s)
|
|
}
|
|
// It answers: healthy, even inside its grace.
|
|
lookOnce(t, j)
|
|
st, _ = j.Look(t.Context())
|
|
if s := stateOf(t, st, "app.web"); s.State != Healthy {
|
|
t.Fatalf("after a passing look: %+v", s)
|
|
}
|
|
// It stops answering, after its grace: one failing look is not yet unhealthy; the second is.
|
|
answering = false
|
|
c.now = t0.Add(time.Minute)
|
|
lookOnce(t, j)
|
|
st, _ = j.Look(t.Context())
|
|
if s := stateOf(t, st, "app.web"); s.State != Healthy {
|
|
t.Fatalf("one failing look made it %s", s.State)
|
|
}
|
|
c.now = c.now.Add(10 * time.Second)
|
|
lookOnce(t, j)
|
|
st, _ = j.Look(t.Context())
|
|
s := stateOf(t, st, "app.web")
|
|
if s.State != Unhealthy || !strings.Contains(s.Reason, "http /healthz on web: answered 503") {
|
|
t.Fatalf("two failing looks: %+v", s)
|
|
}
|
|
// And nothing was restarted, recreated or stopped: the fake runtime was only read.
|
|
answering = true
|
|
c.now = c.now.Add(10 * time.Second)
|
|
lookOnce(t, j)
|
|
st, _ = j.Look(t.Context())
|
|
if s := stateOf(t, st, "app.web"); s.State != Healthy {
|
|
t.Fatalf("answering again: %+v", s)
|
|
}
|
|
}
|
|
|
|
func TestAnHTTPCheckDialsTheEndpointsCurrentPortAfterAPortChange(t *testing.T) {
|
|
hit := map[string]int{}
|
|
handler := func(name string) http.Handler {
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { hit[name]++ })
|
|
}
|
|
before := httptest.NewServer(handler("before"))
|
|
defer before.Close()
|
|
after := httptest.NewServer(handler("after"))
|
|
defer after.Close()
|
|
c := &clock{now: t0}
|
|
rt := &fakeRuntime{containers: map[string]Observed{"web": running("c1", 0, t0.Format(time.RFC3339Nano))}}
|
|
j := aJudge(t, rt, c, "")
|
|
j.Set([]Resource{{Module: "app", ID: "app.web", Kind: KindContainer, Target: "web", Check: httpCheck(portOf(t, before.URL))}})
|
|
j.Look(t.Context())
|
|
lookOnce(t, j)
|
|
// The machine gave the endpoint another port: the next declaration says so, and the check follows.
|
|
j.Set([]Resource{{Module: "app", ID: "app.web", Kind: KindContainer, Target: "web", Check: httpCheck(portOf(t, after.URL))}})
|
|
j.Look(t.Context())
|
|
lookOnce(t, j)
|
|
if hit["before"] != 1 || hit["after"] != 1 {
|
|
t.Fatalf("the check dialled %v; after the port moved it must dial the new one at once", hit)
|
|
}
|
|
}
|
|
|
|
func TestTheRuntimesCheckIsReadAndAnImageWithoutOneIsSaid(t *testing.T) {
|
|
c := &clock{now: t0}
|
|
o := running("c1", 0, t0.Format(time.RFC3339Nano))
|
|
rt := &fakeRuntime{containers: map[string]Observed{"db": o}}
|
|
j := aJudge(t, rt, c, "")
|
|
check := &declaration.Health{Kind: declaration.HealthRuntime, Interval: "30s", Timeout: "5s", Looks: 3, Grace: "1m0s"}
|
|
j.Set([]Resource{{Module: "app", ID: "app.db", Kind: KindContainer, Target: "db", Check: check}})
|
|
look := func(health, said string) State {
|
|
o.Health, o.HealthSaid = health, said
|
|
rt.containers["db"] = o
|
|
st, _ := j.Look(t.Context())
|
|
return stateOf(t, st, "app.db")
|
|
}
|
|
if s := look("starting", ""); s.State != Starting {
|
|
t.Fatalf("the runtime says starting: %+v", s)
|
|
}
|
|
if s := look(Healthy, ""); s.State != Healthy {
|
|
t.Fatalf("the runtime says healthy: %+v", s)
|
|
}
|
|
c.now = t0.Add(2 * time.Minute)
|
|
if s := look(Unhealthy, "curl: (7) Failed to connect to localhost port 3000"); s.State != Unhealthy ||
|
|
!strings.Contains(s.Reason, "Failed to connect to localhost") {
|
|
t.Fatalf("the runtime says unhealthy: %+v", s)
|
|
}
|
|
if s := look("", ""); s.State != Unhealthy || !strings.Contains(s.Reason, "ships no check to adopt") {
|
|
t.Fatalf("an image with no check adopted by name: %+v", s)
|
|
}
|
|
}
|
|
|
|
func TestAUnitIsReadyWhenItsManagerSaysItIsActive(t *testing.T) {
|
|
c := &clock{now: t0}
|
|
rt := &fakeRuntime{units: map[string]Observed{"d.service": {Found: true, Identity: "i1", Running: true}}}
|
|
j := aJudge(t, rt, c, "")
|
|
check := &declaration.Health{Kind: declaration.HealthUnit, Interval: "30s", Timeout: "5s", Looks: 2, Grace: "0s"}
|
|
j.Set([]Resource{{Module: "app", ID: "app.d", Kind: KindService, Target: "d.service", Check: check}})
|
|
st, _ := j.Look(t.Context())
|
|
if s := stateOf(t, st, "app.d"); s.State != Healthy {
|
|
t.Fatalf("an active unit with no grace: %+v", s)
|
|
}
|
|
}
|
|
|
|
func TestNeverMoreLooksThanTheBudget(t *testing.T) {
|
|
j := aJudge(t, &fakeRuntime{}, &clock{now: t0}, "")
|
|
var rs []Resource
|
|
for i := 0; i < 30; i++ {
|
|
rs = append(rs, Resource{Module: "app", ID: "app.c" + strconv.Itoa(i), Kind: KindContainer, Target: "c" + strconv.Itoa(i),
|
|
Check: &declaration.Health{Kind: declaration.HealthTCP, Port: 1, Interval: "10s", Timeout: "1s", Looks: 2, Grace: "0s"}})
|
|
}
|
|
j.Set(rs)
|
|
// 30 checks every 10 s are 180 looks a minute: spaced twice their interval to stay within 90.
|
|
if got := j.Spacing(); got != 2 {
|
|
t.Fatalf("spaced %v", got)
|
|
}
|
|
j.Set(rs[:9])
|
|
if got := j.Spacing(); got != 1 {
|
|
t.Fatalf("54 looks a minute spaced %v", got)
|
|
}
|
|
}
|
|
|
|
func TestAToolCheckIsAskedOfTheNodeTools(t *testing.T) {
|
|
asked := ""
|
|
p := &Probes{AskTool: func(_ context.Context, module, tool string) (bool, string, error) {
|
|
asked = module + "." + tool
|
|
return false, "the admin refused the minted secret", nil
|
|
}}
|
|
ok, why := p.Look(t.Context(), "keycloak", &declaration.Health{Kind: declaration.HealthTool, Tool: "keycloak_admin_health",
|
|
Interval: "30s", Timeout: "5s", Looks: 2, Grace: "0s"})
|
|
if ok || asked != "keycloak.keycloak_admin_health" || !strings.Contains(why, "refused the minted secret") {
|
|
t.Fatalf("asked %q: %v %q", asked, ok, why)
|
|
}
|
|
}
|
|
|
|
// The runtime's state of a container's own check is read out of its whole state: a container that carries
|
|
// none has no Health in it, and that is not an error (a template naming it would refuse every container).
|
|
func TestTheRuntimesCheckIsReadOutOfTheContainersState(t *testing.T) {
|
|
status, said := runtimeHealth(`{"Status":"running","Health":{"Status":"unhealthy","FailingStreak":3,"Log":[` +
|
|
`{"ExitCode":1,"Output":"curl: (7) Failed to connect\n"}]}}`)
|
|
if status != Unhealthy || said != "curl: (7) Failed to connect" {
|
|
t.Errorf("read %q %q", status, said)
|
|
}
|
|
if status, _ := runtimeHealth(`{"Status":"running","Running":true}`); status != "" {
|
|
t.Errorf("a container without a check read as %q", status)
|
|
}
|
|
if !strings.Contains(inspectFormat, "{{json .State}}") || strings.Contains(inspectFormat, ".State.Health") {
|
|
t.Errorf("the inspect names a key a container without a check does not have: %s", inspectFormat)
|
|
}
|
|
}
|
|
|
|
// The engine's own looks run on their own clock, beside the looks of liveness, and are folded in.
|
|
func TestProbeLooksOnItsOwnAndTheNextLookFoldsItIn(t *testing.T) {
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
|
|
defer srv.Close()
|
|
rt := &fakeRuntime{containers: map[string]Observed{"web": running("c1", 0, time.Now().Format(time.RFC3339Nano))}}
|
|
j, err := Open("", rt)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
j.Set([]Resource{{Module: "app", ID: "app.web", Kind: KindContainer, Target: "web", Check: httpCheck(portOf(t, srv.URL))}})
|
|
ctx, cancel := context.WithCancel(t.Context())
|
|
defer cancel()
|
|
go j.Probe(ctx)
|
|
deadline := time.Now().Add(5 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
if st, _ := j.Look(t.Context()); stateOf(t, st, "app.web").State == Healthy {
|
|
return
|
|
}
|
|
time.Sleep(100 * time.Millisecond)
|
|
}
|
|
t.Fatal("the engine's own look was never folded in")
|
|
}
|