jochen
5e189c2fca
mesh/merge-gate pass: builds mesh-host → ace, g14, novox, shanks; no bus step; every machine composes with the change as it did without (4 of 4 compose)
mesh/repo-check pass: THE CHANGE ALTERS ITS OWN CHECK (merge-check.sh): main's version judged it; the change's judges the pull requests after it merges; it…
mesh/delivery delivered
The evidence read "asking it: asking <subject>": the words are now the link's own. A deadline is said as the time the check gave it. A refusal is said only when the bus refused this question, not an earlier one under a grant since widened. merge-check.sh runs the tests that need a bus against a throwaway nats-server when the toolchain has one, and says so when it does not (hq issue 331).
253 lines
10 KiB
Go
253 lines
10 KiB
Go
package liveness
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"net"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-host/internal/declaration"
|
|
)
|
|
|
|
// Readiness, declared (novox/hq ADR 0240 rules 2 and 3, Phase B): starting until its check passed, healthy
|
|
// once it has, unhealthy once its failing looks after the grace reach the declared number; an http check
|
|
// dials the endpoint's current port after a port change; the runtime's check is read, never run by the
|
|
// engine; and never more looks than the budget.
|
|
|
|
func httpCheck(port int) *declaration.Health {
|
|
return &declaration.Health{Kind: declaration.HealthHTTP, Endpoint: "web", Port: port, Path: "/healthz",
|
|
Interval: "10s", Timeout: "2s", Looks: 2, Grace: "30s"}
|
|
}
|
|
|
|
func portOf(t *testing.T, url string) int {
|
|
t.Helper()
|
|
_, p, err := net.SplitHostPort(strings.TrimPrefix(url, "http://"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
n, _ := strconv.Atoi(p)
|
|
return n
|
|
}
|
|
|
|
// lookOnce makes the engine's own due looks at once and waits for them, as Probe would on its tick.
|
|
func lookOnce(t *testing.T, j *Judge) {
|
|
t.Helper()
|
|
for _, d := range j.due() {
|
|
ok, why := j.probes().Look(t.Context(), d.module, d.check)
|
|
j.Record(d.id, d.started, ok, why)
|
|
}
|
|
}
|
|
|
|
func TestAnHTTPCheckIsStartingUntilItPassesAndUnhealthyAfterItsLooksFail(t *testing.T) {
|
|
answering := true
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if !answering || r.URL.Path != "/healthz" {
|
|
w.WriteHeader(http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
w.Write([]byte("ok"))
|
|
}))
|
|
defer srv.Close()
|
|
c := &clock{now: t0}
|
|
rt := &fakeRuntime{containers: map[string]Observed{"web": running("c1", 0, t0.Format(time.RFC3339Nano))}}
|
|
j := aJudge(t, rt, c, filepath.Join(t.TempDir(), FileName))
|
|
web := Resource{Module: "app", ID: "app.web", Kind: KindContainer, Target: "web", Check: httpCheck(portOf(t, srv.URL))}
|
|
j.Set([]Resource{web})
|
|
|
|
// In its grace, before any look: starting.
|
|
st, _ := j.Look(t.Context())
|
|
if s := stateOf(t, st, "app.web"); s.State != Starting || s.CheckOf() != "http" {
|
|
t.Fatalf("before any look: %+v", s)
|
|
}
|
|
// It answers: healthy, even inside its grace.
|
|
lookOnce(t, j)
|
|
st, _ = j.Look(t.Context())
|
|
if s := stateOf(t, st, "app.web"); s.State != Healthy {
|
|
t.Fatalf("after a passing look: %+v", s)
|
|
}
|
|
// It stops answering, after its grace: one failing look is not yet unhealthy; the second is.
|
|
answering = false
|
|
c.now = t0.Add(time.Minute)
|
|
lookOnce(t, j)
|
|
st, _ = j.Look(t.Context())
|
|
if s := stateOf(t, st, "app.web"); s.State != Healthy {
|
|
t.Fatalf("one failing look made it %s", s.State)
|
|
}
|
|
c.now = c.now.Add(10 * time.Second)
|
|
lookOnce(t, j)
|
|
st, _ = j.Look(t.Context())
|
|
s := stateOf(t, st, "app.web")
|
|
if s.State != Unhealthy || !strings.Contains(s.Reason, "http /healthz on web: answered 503") {
|
|
t.Fatalf("two failing looks: %+v", s)
|
|
}
|
|
// And nothing was restarted, recreated or stopped: the fake runtime was only read.
|
|
answering = true
|
|
c.now = c.now.Add(10 * time.Second)
|
|
lookOnce(t, j)
|
|
st, _ = j.Look(t.Context())
|
|
if s := stateOf(t, st, "app.web"); s.State != Healthy {
|
|
t.Fatalf("answering again: %+v", s)
|
|
}
|
|
}
|
|
|
|
func TestAnHTTPCheckDialsTheEndpointsCurrentPortAfterAPortChange(t *testing.T) {
|
|
hit := map[string]int{}
|
|
handler := func(name string) http.Handler {
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { hit[name]++ })
|
|
}
|
|
before := httptest.NewServer(handler("before"))
|
|
defer before.Close()
|
|
after := httptest.NewServer(handler("after"))
|
|
defer after.Close()
|
|
c := &clock{now: t0}
|
|
rt := &fakeRuntime{containers: map[string]Observed{"web": running("c1", 0, t0.Format(time.RFC3339Nano))}}
|
|
j := aJudge(t, rt, c, "")
|
|
j.Set([]Resource{{Module: "app", ID: "app.web", Kind: KindContainer, Target: "web", Check: httpCheck(portOf(t, before.URL))}})
|
|
j.Look(t.Context())
|
|
lookOnce(t, j)
|
|
// The machine gave the endpoint another port: the next declaration says so, and the check follows.
|
|
j.Set([]Resource{{Module: "app", ID: "app.web", Kind: KindContainer, Target: "web", Check: httpCheck(portOf(t, after.URL))}})
|
|
j.Look(t.Context())
|
|
lookOnce(t, j)
|
|
if hit["before"] != 1 || hit["after"] != 1 {
|
|
t.Fatalf("the check dialled %v; after the port moved it must dial the new one at once", hit)
|
|
}
|
|
}
|
|
|
|
func TestTheRuntimesCheckIsReadAndAnImageWithoutOneIsSaid(t *testing.T) {
|
|
c := &clock{now: t0}
|
|
o := running("c1", 0, t0.Format(time.RFC3339Nano))
|
|
rt := &fakeRuntime{containers: map[string]Observed{"db": o}}
|
|
j := aJudge(t, rt, c, "")
|
|
check := &declaration.Health{Kind: declaration.HealthRuntime, Interval: "30s", Timeout: "5s", Looks: 3, Grace: "1m0s"}
|
|
j.Set([]Resource{{Module: "app", ID: "app.db", Kind: KindContainer, Target: "db", Check: check}})
|
|
look := func(health, said string) State {
|
|
o.Health, o.HealthSaid = health, said
|
|
rt.containers["db"] = o
|
|
st, _ := j.Look(t.Context())
|
|
return stateOf(t, st, "app.db")
|
|
}
|
|
if s := look("starting", ""); s.State != Starting {
|
|
t.Fatalf("the runtime says starting: %+v", s)
|
|
}
|
|
if s := look(Healthy, ""); s.State != Healthy {
|
|
t.Fatalf("the runtime says healthy: %+v", s)
|
|
}
|
|
c.now = t0.Add(2 * time.Minute)
|
|
if s := look(Unhealthy, "curl: (7) Failed to connect to localhost port 3000"); s.State != Unhealthy ||
|
|
!strings.Contains(s.Reason, "Failed to connect to localhost") {
|
|
t.Fatalf("the runtime says unhealthy: %+v", s)
|
|
}
|
|
if s := look("", ""); s.State != Unhealthy || !strings.Contains(s.Reason, "ships no check to adopt") {
|
|
t.Fatalf("an image with no check adopted by name: %+v", s)
|
|
}
|
|
}
|
|
|
|
func TestAUnitIsReadyWhenItsManagerSaysItIsActive(t *testing.T) {
|
|
c := &clock{now: t0}
|
|
rt := &fakeRuntime{units: map[string]Observed{"d.service": {Found: true, Identity: "i1", Running: true}}}
|
|
j := aJudge(t, rt, c, "")
|
|
check := &declaration.Health{Kind: declaration.HealthUnit, Interval: "30s", Timeout: "5s", Looks: 2, Grace: "0s"}
|
|
j.Set([]Resource{{Module: "app", ID: "app.d", Kind: KindService, Target: "d.service", Check: check}})
|
|
st, _ := j.Look(t.Context())
|
|
if s := stateOf(t, st, "app.d"); s.State != Healthy {
|
|
t.Fatalf("an active unit with no grace: %+v", s)
|
|
}
|
|
}
|
|
|
|
func TestNeverMoreLooksThanTheBudget(t *testing.T) {
|
|
j := aJudge(t, &fakeRuntime{}, &clock{now: t0}, "")
|
|
var rs []Resource
|
|
for i := 0; i < 30; i++ {
|
|
rs = append(rs, Resource{Module: "app", ID: "app.c" + strconv.Itoa(i), Kind: KindContainer, Target: "c" + strconv.Itoa(i),
|
|
Check: &declaration.Health{Kind: declaration.HealthTCP, Port: 1, Interval: "10s", Timeout: "1s", Looks: 2, Grace: "0s"}})
|
|
}
|
|
j.Set(rs)
|
|
// 30 checks every 10 s are 180 looks a minute: spaced twice their interval to stay within 90.
|
|
if got := j.Spacing(); got != 2 {
|
|
t.Fatalf("spaced %v", got)
|
|
}
|
|
j.Set(rs[:9])
|
|
if got := j.Spacing(); got != 1 {
|
|
t.Fatalf("54 looks a minute spaced %v", got)
|
|
}
|
|
}
|
|
|
|
func TestAToolCheckIsAskedOfTheNodeTools(t *testing.T) {
|
|
asked := ""
|
|
p := &Probes{AskTool: func(_ context.Context, module, tool string) (bool, string, error) {
|
|
asked = module + "." + tool
|
|
return false, "the admin refused the minted secret", nil
|
|
}}
|
|
ok, why := p.Look(t.Context(), "keycloak", &declaration.Health{Kind: declaration.HealthTool, Tool: "keycloak_admin_health",
|
|
Interval: "30s", Timeout: "5s", Looks: 2, Grace: "0s"})
|
|
if ok || asked != "keycloak.keycloak_admin_health" || !strings.Contains(why, "refused the minted secret") {
|
|
t.Fatalf("asked %q: %v %q", asked, ok, why)
|
|
}
|
|
}
|
|
|
|
// A tool that does not answer in time is said as the time it was given, once, and an error from the link is
|
|
// said as it is, without a second "asking" in front of what the link already says.
|
|
func TestAToolCheckThatGetsNoAnswerSaysHowLongItWaited(t *testing.T) {
|
|
check := &declaration.Health{Kind: declaration.HealthTool, Tool: "sensors_health",
|
|
Interval: "30s", Timeout: "5s", Looks: 2, Grace: "0s"}
|
|
p := &Probes{AskTool: func(context.Context, string, string) (bool, string, error) {
|
|
return false, "", fmt.Errorf("no answer from mesh.mod.sensors.tool.sensors_health.anchor: %w", context.DeadlineExceeded)
|
|
}}
|
|
if ok, why := p.Look(t.Context(), "sensors", check); ok || why != "no answer within 5s" {
|
|
t.Fatalf("%v %q", ok, why)
|
|
}
|
|
p.AskTool = func(context.Context, string, string) (bool, string, error) {
|
|
return false, "", errors.New("no link to the bus is open")
|
|
}
|
|
if ok, why := p.Look(t.Context(), "sensors", check); ok || why != "no link to the bus is open" {
|
|
t.Fatalf("%v %q", ok, why)
|
|
}
|
|
}
|
|
|
|
// The runtime's state of a container's own check is read out of its whole state: a container that carries
|
|
// none has no Health in it, and that is not an error (a template naming it would refuse every container).
|
|
func TestTheRuntimesCheckIsReadOutOfTheContainersState(t *testing.T) {
|
|
status, said := runtimeHealth(`{"Status":"running","Health":{"Status":"unhealthy","FailingStreak":3,"Log":[` +
|
|
`{"ExitCode":1,"Output":"curl: (7) Failed to connect\n"}]}}`)
|
|
if status != Unhealthy || said != "curl: (7) Failed to connect" {
|
|
t.Errorf("read %q %q", status, said)
|
|
}
|
|
if status, _ := runtimeHealth(`{"Status":"running","Running":true}`); status != "" {
|
|
t.Errorf("a container without a check read as %q", status)
|
|
}
|
|
if !strings.Contains(inspectFormat, "{{json .State}}") || strings.Contains(inspectFormat, ".State.Health") {
|
|
t.Errorf("the inspect names a key a container without a check does not have: %s", inspectFormat)
|
|
}
|
|
}
|
|
|
|
// The engine's own looks run on their own clock, beside the looks of liveness, and are folded in.
|
|
func TestProbeLooksOnItsOwnAndTheNextLookFoldsItIn(t *testing.T) {
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
|
|
defer srv.Close()
|
|
rt := &fakeRuntime{containers: map[string]Observed{"web": running("c1", 0, time.Now().Format(time.RFC3339Nano))}}
|
|
j, err := Open("", rt)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
j.Set([]Resource{{Module: "app", ID: "app.web", Kind: KindContainer, Target: "web", Check: httpCheck(portOf(t, srv.URL))}})
|
|
ctx, cancel := context.WithCancel(t.Context())
|
|
defer cancel()
|
|
go j.Probe(ctx)
|
|
deadline := time.Now().Add(5 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
if st, _ := j.Look(t.Context()); stateOf(t, st, "app.web").State == Healthy {
|
|
return
|
|
}
|
|
time.Sleep(100 * time.Millisecond)
|
|
}
|
|
t.Fatal("the engine's own look was never folded in")
|
|
}
|