package liveness import ( "context" "encoding/json" "path/filepath" "strings" "testing" "time" "git.novox.be/novox/mesh-sdk/go/health" "github.com/novox/mesh-host/internal/declaration" ) // A module's tool check may say it waits for the operator (novox/hq ADR 0283, issue 386): a named secret or // setting not given yet, and nothing else wrong. The engine states that resource waiting, with its waits, on // one look, inside the grace too; the next look that says anything else replaces it. var password = health.Wait{Part: "the source games", Secret: "smb-password-games", What: "the password of the source games"} var toolCheck = &declaration.Health{Kind: declaration.HealthTool, Tool: "mounts_health", Interval: "30s", Timeout: "5s", Looks: 2, Grace: "1m30s"} // toolJudge is a judge of one running process whose tool check answers what *answer holds. func toolJudge(t *testing.T, answer *[]byte) (*Judge, *clock) { j, c, _, _ := toolJudgeAt(t, answer, filepath.Join(t.TempDir(), FileName)) return j, c } // toolJudgeAt is toolJudge keeping its file at path, with its runtime. func toolJudgeAt(t *testing.T, answer *[]byte, path string) (*Judge, *clock, *fakeRuntime, string) { t.Helper() c := &clock{now: t0} rt := &fakeRuntime{containers: map[string]Observed{"watch": running("c1", 0, t0.Format(time.RFC3339Nano))}} j := aJudge(t, rt, c, path) j.Probes = &Probes{AskTool: func(context.Context, string, string) (health.Answer, error) { return health.Read(*answer) }} j.Set([]Resource{{Module: "mounts", ID: "mounts.watch", Kind: KindContainer, Target: "watch", Check: toolCheck}}) j.Look(t.Context()) return j, c, rt, path } func answerOf(t *testing.T, a health.Answer) []byte { t.Helper() b, err := json.Marshal(a) if err != nil { t.Fatal(err) } return b } func next(t *testing.T, j *Judge, c *clock, by time.Duration) State { t.Helper() c.now = c.now.Add(by) lookOnce(t, j) st, _ := j.Look(t.Context()) return stateOf(t, st, "mounts.watch") } func TestAToolThatWaitsIsStatedWaitingWithItsWaitsInsideTheGrace(t *testing.T) { answer := answerOf(t, health.Waiting("the source games waits for its password", password)) j, c := toolJudge(t, &answer) s := next(t, j, c, 0) if s.State != Waiting || len(s.Waits) != 1 || s.Waits[0].Secret != "smb-password-games" || !strings.Contains(s.Reason, "waits for the operator: the source games waits for its password") { t.Fatalf("in its grace, a tool that waits: %+v", s) } if s.Streak != 0 { t.Fatalf("a wait counted as a failing streak: %+v", s) } } func TestAToolThatWaitsStaysWaitingAfterTheGraceAndIsNoFailingLook(t *testing.T) { answer := answerOf(t, health.Waiting("waits", password)) j, c := toolJudge(t, &answer) for i := 0; i < 5; i++ { if s := next(t, j, c, 30*time.Second); s.State != Waiting { t.Fatalf("look %d after the grace: %+v", i, s) } } } func TestTheNextHealthyAnswerAfterAWaitIsHealthy(t *testing.T) { answer := answerOf(t, health.Waiting("waits", password)) j, c := toolJudge(t, &answer) next(t, j, c, 2*time.Minute) answer = answerOf(t, health.Healthy("every source armed")) if s := next(t, j, c, 30*time.Second); s.State != Healthy || len(s.Waits) != 0 { t.Fatalf("healthy after a wait: %+v", s) } } func TestAnUnhealthyAnswerAfterAWaitFollowsTheFailingLooks(t *testing.T) { answer := answerOf(t, health.Waiting("waits", password)) j, c := toolJudge(t, &answer) next(t, j, c, 2*time.Minute) answer = answerOf(t, health.Unhealthy("the share is unreachable")) s := next(t, j, c, 30*time.Second) if s.State == Unhealthy || s.State == Waiting || s.State == Healthy || len(s.Waits) != 0 { t.Fatalf("one failing look after a wait made it %+v", s) } if s = next(t, j, c, 30*time.Second); s.State != Unhealthy || !strings.Contains(s.Reason, "the share is unreachable") { t.Fatalf("two failing looks after a wait: %+v", s) } } func TestAnAnswerWithoutStateIsJudgedAsBefore(t *testing.T) { answer := []byte(`{"healthy":false,"why":"down"}`) j, c := toolJudge(t, &answer) next(t, j, c, 2*time.Minute) if s := next(t, j, c, 30*time.Second); s.State != Unhealthy || len(s.Waits) != 0 { t.Fatalf("an old unhealthy answer: %+v", s) } answer = []byte(`{"healthy":true,"why":"ok"}`) if s := next(t, j, c, 30*time.Second); s.State != Healthy { t.Fatalf("an old healthy answer: %+v", s) } } func TestAMalformedWaitIsUnhealthyAndExcusesNothing(t *testing.T) { answer := []byte(`{"state":"waits","healthy":false,"why":"it waits","waits":[{"part":"p","what":"the password"}]}`) j, c := toolJudge(t, &answer) next(t, j, c, 2*time.Minute) s := next(t, j, c, 30*time.Second) if s.State != Unhealthy || len(s.Waits) != 0 || !strings.Contains(s.Reason, "neither a secret nor a setting") { t.Fatalf("a wait naming nothing: %+v", s) } } // A pass from before a wait is not kept through it: passed, waited, then one failing look after the grace is // not healthy. func TestAPassBeforeAWaitIsNotKeptThroughIt(t *testing.T) { answer := answerOf(t, health.Healthy("ok")) j, c := toolJudge(t, &answer) if s := next(t, j, c, 0); s.State != Healthy { t.Fatalf("passing: %+v", s) } answer = answerOf(t, health.Waiting("waits", password)) next(t, j, c, 2*time.Minute) answer = answerOf(t, health.Unhealthy("the share is unreachable")) if s := next(t, j, c, 30*time.Second); s.State == Healthy || s.State == Waiting { t.Fatalf("one failing look after a wait that followed a pass: %+v", s) } } // A resource that is down is down, whatever its last look said it waits for. func TestDownBeatsAWait(t *testing.T) { answer := answerOf(t, health.Waiting("waits", password)) j, c, rt, _ := toolJudgeAt(t, &answer, filepath.Join(t.TempDir(), FileName)) next(t, j, c, 2*time.Minute) rt.containers["watch"] = Observed{Found: true, Identity: "c1", Started: t0.Format(time.RFC3339Nano)} c.now = c.now.Add(15 * time.Second) st, _ := j.Look(t.Context()) if s := stateOf(t, st, "mounts.watch"); s.State != Unhealthy || s.Reason != ReasonDown || len(s.Waits) != 0 { t.Fatalf("a stopped process that waited: %+v", s) } } // A changed check says nothing of what the old one found waiting. func TestAChangedCheckClearsTheWaits(t *testing.T) { answer := answerOf(t, health.Waiting("waits", password)) j, c := toolJudge(t, &answer) next(t, j, c, 2*time.Minute) other := *toolCheck other.Interval = "60s" j.Set([]Resource{{Module: "mounts", ID: "mounts.watch", Kind: KindContainer, Target: "watch", Check: &other}}) st, _ := j.Look(t.Context()) if s := stateOf(t, st, "mounts.watch"); s.State == Waiting || len(s.Waits) != 0 { t.Fatalf("after its check changed: %+v", s) } } // The engine restarted keeps what a resource waits for, as it keeps everything else it judged. func TestWaitsAreKeptAcrossTheEnginesRestart(t *testing.T) { answer := answerOf(t, health.Waiting("waits", password)) path := filepath.Join(t.TempDir(), FileName) j, c, rt, _ := toolJudgeAt(t, &answer, path) next(t, j, c, 2*time.Minute) again := aJudge(t, rt, c, path) again.Set([]Resource{{Module: "mounts", ID: "mounts.watch", Kind: KindContainer, Target: "watch", Check: toolCheck}}) st, _ := again.Look(t.Context()) if s := stateOf(t, st, "mounts.watch"); s.State != Waiting || len(s.Waits) != 1 || s.Waits[0].Secret != password.Secret { t.Fatalf("after the engine restarted: %+v", s) } } // Waiting and healthy each way is a change, and so is a change of what it waits for alone. func TestAChangeOfWaitingOrOfItsWaitsIsAChange(t *testing.T) { answer := answerOf(t, health.Healthy("ok")) j, c := toolJudge(t, &answer) look := func(by time.Duration) bool { c.now = c.now.Add(by) lookOnce(t, j) _, changed := j.Look(t.Context()) return changed } look(0) answer = answerOf(t, health.Waiting("waits", password)) if !look(30 * time.Second) { t.Fatal("healthy to waiting was no change") } if look(30 * time.Second) { t.Fatal("the same wait again was a change") } user := health.Wait{Part: "the source games", Setting: "smb-users", What: "the username of the source games"} answer = answerOf(t, health.Waiting("waits", password, user)) if !look(30 * time.Second) { t.Fatal("a wait added was no change") } answer = answerOf(t, health.Healthy("ok")) if !look(30 * time.Second) { t.Fatal("waiting to healthy was no change") } }