package liveness import ( "context" "net" "net/http" "net/http/httptest" "path/filepath" "strconv" "strings" "testing" "time" "github.com/novox/mesh-host/internal/declaration" ) // Readiness, declared (novox/hq ADR 0240 rules 2 and 3, Phase B): starting until its check passed, healthy // once it has, unhealthy once its failing looks after the grace reach the declared number; an http check // dials the endpoint's current port after a port change; the runtime's check is read, never run by the // engine; and never more looks than the budget. func httpCheck(port int) *declaration.Health { return &declaration.Health{Kind: declaration.HealthHTTP, Endpoint: "web", Port: port, Path: "/healthz", Interval: "10s", Timeout: "2s", Looks: 2, Grace: "30s"} } func portOf(t *testing.T, url string) int { t.Helper() _, p, err := net.SplitHostPort(strings.TrimPrefix(url, "http://")) if err != nil { t.Fatal(err) } n, _ := strconv.Atoi(p) return n } // lookOnce makes the engine's own due looks at once and waits for them, as Probe would on its tick. func lookOnce(t *testing.T, j *Judge) { t.Helper() for _, d := range j.due() { ok, why := j.probes().Look(t.Context(), d.module, d.check) j.Record(d.id, d.started, ok, why) } } func TestAnHTTPCheckIsStartingUntilItPassesAndUnhealthyAfterItsLooksFail(t *testing.T) { answering := true srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if !answering || r.URL.Path != "/healthz" { w.WriteHeader(http.StatusServiceUnavailable) return } w.Write([]byte("ok")) })) defer srv.Close() c := &clock{now: t0} rt := &fakeRuntime{containers: map[string]Observed{"web": running("c1", 0, t0.Format(time.RFC3339Nano))}} j := aJudge(t, rt, c, filepath.Join(t.TempDir(), FileName)) web := Resource{Module: "app", ID: "app.web", Kind: KindContainer, Target: "web", Check: httpCheck(portOf(t, srv.URL))} j.Set([]Resource{web}) // In its grace, before any look: starting. st, _ := j.Look(t.Context()) if s := stateOf(t, st, "app.web"); s.State != Starting || s.CheckOf() != "http" { t.Fatalf("before any look: %+v", s) } // It answers: healthy, even inside its grace. lookOnce(t, j) st, _ = j.Look(t.Context()) if s := stateOf(t, st, "app.web"); s.State != Healthy { t.Fatalf("after a passing look: %+v", s) } // It stops answering, after its grace: one failing look is not yet unhealthy; the second is. answering = false c.now = t0.Add(time.Minute) lookOnce(t, j) st, _ = j.Look(t.Context()) if s := stateOf(t, st, "app.web"); s.State != Healthy { t.Fatalf("one failing look made it %s", s.State) } c.now = c.now.Add(10 * time.Second) lookOnce(t, j) st, _ = j.Look(t.Context()) s := stateOf(t, st, "app.web") if s.State != Unhealthy || !strings.Contains(s.Reason, "http /healthz on web: answered 503") { t.Fatalf("two failing looks: %+v", s) } // And nothing was restarted, recreated or stopped: the fake runtime was only read. answering = true c.now = c.now.Add(10 * time.Second) lookOnce(t, j) st, _ = j.Look(t.Context()) if s := stateOf(t, st, "app.web"); s.State != Healthy { t.Fatalf("answering again: %+v", s) } } func TestAnHTTPCheckDialsTheEndpointsCurrentPortAfterAPortChange(t *testing.T) { hit := map[string]int{} handler := func(name string) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { hit[name]++ }) } before := httptest.NewServer(handler("before")) defer before.Close() after := httptest.NewServer(handler("after")) defer after.Close() c := &clock{now: t0} rt := &fakeRuntime{containers: map[string]Observed{"web": running("c1", 0, t0.Format(time.RFC3339Nano))}} j := aJudge(t, rt, c, "") j.Set([]Resource{{Module: "app", ID: "app.web", Kind: KindContainer, Target: "web", Check: httpCheck(portOf(t, before.URL))}}) j.Look(t.Context()) lookOnce(t, j) // The machine gave the endpoint another port: the next declaration says so, and the check follows. j.Set([]Resource{{Module: "app", ID: "app.web", Kind: KindContainer, Target: "web", Check: httpCheck(portOf(t, after.URL))}}) j.Look(t.Context()) lookOnce(t, j) if hit["before"] != 1 || hit["after"] != 1 { t.Fatalf("the check dialled %v; after the port moved it must dial the new one at once", hit) } } func TestTheRuntimesCheckIsReadAndAnImageWithoutOneIsSaid(t *testing.T) { c := &clock{now: t0} o := running("c1", 0, t0.Format(time.RFC3339Nano)) rt := &fakeRuntime{containers: map[string]Observed{"db": o}} j := aJudge(t, rt, c, "") check := &declaration.Health{Kind: declaration.HealthRuntime, Interval: "30s", Timeout: "5s", Looks: 3, Grace: "1m0s"} j.Set([]Resource{{Module: "app", ID: "app.db", Kind: KindContainer, Target: "db", Check: check}}) look := func(health, said string) State { o.Health, o.HealthSaid = health, said rt.containers["db"] = o st, _ := j.Look(t.Context()) return stateOf(t, st, "app.db") } if s := look("starting", ""); s.State != Starting { t.Fatalf("the runtime says starting: %+v", s) } if s := look(Healthy, ""); s.State != Healthy { t.Fatalf("the runtime says healthy: %+v", s) } c.now = t0.Add(2 * time.Minute) if s := look(Unhealthy, "curl: (7) Failed to connect to localhost port 3000"); s.State != Unhealthy || !strings.Contains(s.Reason, "Failed to connect to localhost") { t.Fatalf("the runtime says unhealthy: %+v", s) } if s := look("", ""); s.State != Unhealthy || !strings.Contains(s.Reason, "ships no check to adopt") { t.Fatalf("an image with no check adopted by name: %+v", s) } } func TestAUnitIsReadyWhenItsManagerSaysItIsActive(t *testing.T) { c := &clock{now: t0} rt := &fakeRuntime{units: map[string]Observed{"d.service": {Found: true, Identity: "i1", Running: true}}} j := aJudge(t, rt, c, "") check := &declaration.Health{Kind: declaration.HealthUnit, Interval: "30s", Timeout: "5s", Looks: 2, Grace: "0s"} j.Set([]Resource{{Module: "app", ID: "app.d", Kind: KindService, Target: "d.service", Check: check}}) st, _ := j.Look(t.Context()) if s := stateOf(t, st, "app.d"); s.State != Healthy { t.Fatalf("an active unit with no grace: %+v", s) } } func TestNeverMoreLooksThanTheBudget(t *testing.T) { j := aJudge(t, &fakeRuntime{}, &clock{now: t0}, "") var rs []Resource for i := 0; i < 30; i++ { rs = append(rs, Resource{Module: "app", ID: "app.c" + strconv.Itoa(i), Kind: KindContainer, Target: "c" + strconv.Itoa(i), Check: &declaration.Health{Kind: declaration.HealthTCP, Port: 1, Interval: "10s", Timeout: "1s", Looks: 2, Grace: "0s"}}) } j.Set(rs) // 30 checks every 10 s are 180 looks a minute: spaced twice their interval to stay within 90. if got := j.Spacing(); got != 2 { t.Fatalf("spaced %v", got) } j.Set(rs[:9]) if got := j.Spacing(); got != 1 { t.Fatalf("54 looks a minute spaced %v", got) } } func TestAToolCheckIsAskedOfTheNodeTools(t *testing.T) { asked := "" p := &Probes{AskTool: func(_ context.Context, module, tool string) (bool, string, error) { asked = module + "." + tool return false, "the admin refused the minted secret", nil }} ok, why := p.Look(t.Context(), "keycloak", &declaration.Health{Kind: declaration.HealthTool, Tool: "keycloak_admin_health", Interval: "30s", Timeout: "5s", Looks: 2, Grace: "0s"}) if ok || asked != "keycloak.keycloak_admin_health" || !strings.Contains(why, "refused the minted secret") { t.Fatalf("asked %q: %v %q", asked, ok, why) } } // The runtime's state of a container's own check is read out of its whole state: a container that carries // none has no Health in it, and that is not an error (a template naming it would refuse every container). func TestTheRuntimesCheckIsReadOutOfTheContainersState(t *testing.T) { status, said := runtimeHealth(`{"Status":"running","Health":{"Status":"unhealthy","FailingStreak":3,"Log":[` + `{"ExitCode":1,"Output":"curl: (7) Failed to connect\n"}]}}`) if status != Unhealthy || said != "curl: (7) Failed to connect" { t.Errorf("read %q %q", status, said) } if status, _ := runtimeHealth(`{"Status":"running","Running":true}`); status != "" { t.Errorf("a container without a check read as %q", status) } if !strings.Contains(inspectFormat, "{{json .State}}") || strings.Contains(inspectFormat, ".State.Health") { t.Errorf("the inspect names a key a container without a check does not have: %s", inspectFormat) } } // The engine's own looks run on their own clock, beside the looks of liveness, and are folded in. func TestProbeLooksOnItsOwnAndTheNextLookFoldsItIn(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) defer srv.Close() rt := &fakeRuntime{containers: map[string]Observed{"web": running("c1", 0, time.Now().Format(time.RFC3339Nano))}} j, err := Open("", rt) if err != nil { t.Fatal(err) } j.Set([]Resource{{Module: "app", ID: "app.web", Kind: KindContainer, Target: "web", Check: httpCheck(portOf(t, srv.URL))}}) ctx, cancel := context.WithCancel(t.Context()) defer cancel() go j.Probe(ctx) deadline := time.Now().Add(5 * time.Second) for time.Now().Before(deadline) { if st, _ := j.Look(t.Context()); stateOf(t, st, "app.web").State == Healthy { return } time.Sleep(100 * time.Millisecond) } t.Fatal("the engine's own look was never folded in") }