jochen
9545121ec9
A resource that passed, then waited, then failed once after its grace was still stated healthy from the old pass; and a change of what it waits for alone raised no change event, so the controller could miss it.
216 lines
8.1 KiB
Go
216 lines
8.1 KiB
Go
package liveness
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/health"
|
|
"github.com/novox/mesh-host/internal/declaration"
|
|
)
|
|
|
|
// A module's tool check may say it waits for the operator (novox/hq ADR 0283, issue 386): a named secret or
|
|
// setting not given yet, and nothing else wrong. The engine states that resource waiting, with its waits, on
|
|
// one look, inside the grace too; the next look that says anything else replaces it.
|
|
|
|
var password = health.Wait{Part: "the source games", Secret: "smb-password-games", What: "the password of the source games"}
|
|
|
|
var toolCheck = &declaration.Health{Kind: declaration.HealthTool, Tool: "mounts_health", Interval: "30s", Timeout: "5s",
|
|
Looks: 2, Grace: "1m30s"}
|
|
|
|
// toolJudge is a judge of one running process whose tool check answers what *answer holds.
|
|
func toolJudge(t *testing.T, answer *[]byte) (*Judge, *clock) {
|
|
j, c, _, _ := toolJudgeAt(t, answer, filepath.Join(t.TempDir(), FileName))
|
|
return j, c
|
|
}
|
|
|
|
// toolJudgeAt is toolJudge keeping its file at path, with its runtime.
|
|
func toolJudgeAt(t *testing.T, answer *[]byte, path string) (*Judge, *clock, *fakeRuntime, string) {
|
|
t.Helper()
|
|
c := &clock{now: t0}
|
|
rt := &fakeRuntime{containers: map[string]Observed{"watch": running("c1", 0, t0.Format(time.RFC3339Nano))}}
|
|
j := aJudge(t, rt, c, path)
|
|
j.Probes = &Probes{AskTool: func(context.Context, string, string) (health.Answer, error) {
|
|
return health.Read(*answer)
|
|
}}
|
|
j.Set([]Resource{{Module: "mounts", ID: "mounts.watch", Kind: KindContainer, Target: "watch", Check: toolCheck}})
|
|
j.Look(t.Context())
|
|
return j, c, rt, path
|
|
}
|
|
|
|
func answerOf(t *testing.T, a health.Answer) []byte {
|
|
t.Helper()
|
|
b, err := json.Marshal(a)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return b
|
|
}
|
|
|
|
func next(t *testing.T, j *Judge, c *clock, by time.Duration) State {
|
|
t.Helper()
|
|
c.now = c.now.Add(by)
|
|
lookOnce(t, j)
|
|
st, _ := j.Look(t.Context())
|
|
return stateOf(t, st, "mounts.watch")
|
|
}
|
|
|
|
func TestAToolThatWaitsIsStatedWaitingWithItsWaitsInsideTheGrace(t *testing.T) {
|
|
answer := answerOf(t, health.Waiting("the source games waits for its password", password))
|
|
j, c := toolJudge(t, &answer)
|
|
s := next(t, j, c, 0)
|
|
if s.State != Waiting || len(s.Waits) != 1 || s.Waits[0].Secret != "smb-password-games" ||
|
|
!strings.Contains(s.Reason, "waits for the operator: the source games waits for its password") {
|
|
t.Fatalf("in its grace, a tool that waits: %+v", s)
|
|
}
|
|
if s.Streak != 0 {
|
|
t.Fatalf("a wait counted as a failing streak: %+v", s)
|
|
}
|
|
}
|
|
|
|
func TestAToolThatWaitsStaysWaitingAfterTheGraceAndIsNoFailingLook(t *testing.T) {
|
|
answer := answerOf(t, health.Waiting("waits", password))
|
|
j, c := toolJudge(t, &answer)
|
|
for i := 0; i < 5; i++ {
|
|
if s := next(t, j, c, 30*time.Second); s.State != Waiting {
|
|
t.Fatalf("look %d after the grace: %+v", i, s)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestTheNextHealthyAnswerAfterAWaitIsHealthy(t *testing.T) {
|
|
answer := answerOf(t, health.Waiting("waits", password))
|
|
j, c := toolJudge(t, &answer)
|
|
next(t, j, c, 2*time.Minute)
|
|
answer = answerOf(t, health.Healthy("every source armed"))
|
|
if s := next(t, j, c, 30*time.Second); s.State != Healthy || len(s.Waits) != 0 {
|
|
t.Fatalf("healthy after a wait: %+v", s)
|
|
}
|
|
}
|
|
|
|
func TestAnUnhealthyAnswerAfterAWaitFollowsTheFailingLooks(t *testing.T) {
|
|
answer := answerOf(t, health.Waiting("waits", password))
|
|
j, c := toolJudge(t, &answer)
|
|
next(t, j, c, 2*time.Minute)
|
|
answer = answerOf(t, health.Unhealthy("the share is unreachable"))
|
|
s := next(t, j, c, 30*time.Second)
|
|
if s.State == Unhealthy || s.State == Waiting || s.State == Healthy || len(s.Waits) != 0 {
|
|
t.Fatalf("one failing look after a wait made it %+v", s)
|
|
}
|
|
if s = next(t, j, c, 30*time.Second); s.State != Unhealthy || !strings.Contains(s.Reason, "the share is unreachable") {
|
|
t.Fatalf("two failing looks after a wait: %+v", s)
|
|
}
|
|
}
|
|
|
|
func TestAnAnswerWithoutStateIsJudgedAsBefore(t *testing.T) {
|
|
answer := []byte(`{"healthy":false,"why":"down"}`)
|
|
j, c := toolJudge(t, &answer)
|
|
next(t, j, c, 2*time.Minute)
|
|
if s := next(t, j, c, 30*time.Second); s.State != Unhealthy || len(s.Waits) != 0 {
|
|
t.Fatalf("an old unhealthy answer: %+v", s)
|
|
}
|
|
answer = []byte(`{"healthy":true,"why":"ok"}`)
|
|
if s := next(t, j, c, 30*time.Second); s.State != Healthy {
|
|
t.Fatalf("an old healthy answer: %+v", s)
|
|
}
|
|
}
|
|
|
|
func TestAMalformedWaitIsUnhealthyAndExcusesNothing(t *testing.T) {
|
|
answer := []byte(`{"state":"waits","healthy":false,"why":"it waits","waits":[{"part":"p","what":"the password"}]}`)
|
|
j, c := toolJudge(t, &answer)
|
|
next(t, j, c, 2*time.Minute)
|
|
s := next(t, j, c, 30*time.Second)
|
|
if s.State != Unhealthy || len(s.Waits) != 0 || !strings.Contains(s.Reason, "neither a secret nor a setting") {
|
|
t.Fatalf("a wait naming nothing: %+v", s)
|
|
}
|
|
}
|
|
|
|
// A pass from before a wait is not kept through it: passed, waited, then one failing look after the grace is
|
|
// not healthy.
|
|
func TestAPassBeforeAWaitIsNotKeptThroughIt(t *testing.T) {
|
|
answer := answerOf(t, health.Healthy("ok"))
|
|
j, c := toolJudge(t, &answer)
|
|
if s := next(t, j, c, 0); s.State != Healthy {
|
|
t.Fatalf("passing: %+v", s)
|
|
}
|
|
answer = answerOf(t, health.Waiting("waits", password))
|
|
next(t, j, c, 2*time.Minute)
|
|
answer = answerOf(t, health.Unhealthy("the share is unreachable"))
|
|
if s := next(t, j, c, 30*time.Second); s.State == Healthy || s.State == Waiting {
|
|
t.Fatalf("one failing look after a wait that followed a pass: %+v", s)
|
|
}
|
|
}
|
|
|
|
// A resource that is down is down, whatever its last look said it waits for.
|
|
func TestDownBeatsAWait(t *testing.T) {
|
|
answer := answerOf(t, health.Waiting("waits", password))
|
|
j, c, rt, _ := toolJudgeAt(t, &answer, filepath.Join(t.TempDir(), FileName))
|
|
next(t, j, c, 2*time.Minute)
|
|
rt.containers["watch"] = Observed{Found: true, Identity: "c1", Started: t0.Format(time.RFC3339Nano)}
|
|
c.now = c.now.Add(15 * time.Second)
|
|
st, _ := j.Look(t.Context())
|
|
if s := stateOf(t, st, "mounts.watch"); s.State != Unhealthy || s.Reason != ReasonDown || len(s.Waits) != 0 {
|
|
t.Fatalf("a stopped process that waited: %+v", s)
|
|
}
|
|
}
|
|
|
|
// A changed check says nothing of what the old one found waiting.
|
|
func TestAChangedCheckClearsTheWaits(t *testing.T) {
|
|
answer := answerOf(t, health.Waiting("waits", password))
|
|
j, c := toolJudge(t, &answer)
|
|
next(t, j, c, 2*time.Minute)
|
|
other := *toolCheck
|
|
other.Interval = "60s"
|
|
j.Set([]Resource{{Module: "mounts", ID: "mounts.watch", Kind: KindContainer, Target: "watch", Check: &other}})
|
|
st, _ := j.Look(t.Context())
|
|
if s := stateOf(t, st, "mounts.watch"); s.State == Waiting || len(s.Waits) != 0 {
|
|
t.Fatalf("after its check changed: %+v", s)
|
|
}
|
|
}
|
|
|
|
// The engine restarted keeps what a resource waits for, as it keeps everything else it judged.
|
|
func TestWaitsAreKeptAcrossTheEnginesRestart(t *testing.T) {
|
|
answer := answerOf(t, health.Waiting("waits", password))
|
|
path := filepath.Join(t.TempDir(), FileName)
|
|
j, c, rt, _ := toolJudgeAt(t, &answer, path)
|
|
next(t, j, c, 2*time.Minute)
|
|
again := aJudge(t, rt, c, path)
|
|
again.Set([]Resource{{Module: "mounts", ID: "mounts.watch", Kind: KindContainer, Target: "watch", Check: toolCheck}})
|
|
st, _ := again.Look(t.Context())
|
|
if s := stateOf(t, st, "mounts.watch"); s.State != Waiting || len(s.Waits) != 1 || s.Waits[0].Secret != password.Secret {
|
|
t.Fatalf("after the engine restarted: %+v", s)
|
|
}
|
|
}
|
|
|
|
// Waiting and healthy each way is a change, and so is a change of what it waits for alone.
|
|
func TestAChangeOfWaitingOrOfItsWaitsIsAChange(t *testing.T) {
|
|
answer := answerOf(t, health.Healthy("ok"))
|
|
j, c := toolJudge(t, &answer)
|
|
look := func(by time.Duration) bool {
|
|
c.now = c.now.Add(by)
|
|
lookOnce(t, j)
|
|
_, changed := j.Look(t.Context())
|
|
return changed
|
|
}
|
|
look(0)
|
|
answer = answerOf(t, health.Waiting("waits", password))
|
|
if !look(30 * time.Second) {
|
|
t.Fatal("healthy to waiting was no change")
|
|
}
|
|
if look(30 * time.Second) {
|
|
t.Fatal("the same wait again was a change")
|
|
}
|
|
user := health.Wait{Part: "the source games", Setting: "smb-users", What: "the username of the source games"}
|
|
answer = answerOf(t, health.Waiting("waits", password, user))
|
|
if !look(30 * time.Second) {
|
|
t.Fatal("a wait added was no change")
|
|
}
|
|
answer = answerOf(t, health.Healthy("ok"))
|
|
if !look(30 * time.Second) {
|
|
t.Fatal("waiting to healthy was no change")
|
|
}
|
|
}
|