mesh-host/internal/liveness/waiting_test.go

package liveness

import (
	"context"
	"encoding/json"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"git.novox.be/novox/mesh-sdk/go/health"
	"github.com/novox/mesh-host/internal/declaration"
)

// A module's tool check may say it waits for the operator (novox/hq ADR 0283, issue 386): a named secret or
// setting not given yet, and nothing else wrong. The engine states that resource waiting, with its waits, on
// one look, inside the grace too; the next look that says anything else replaces it.

var password = health.Wait{Part: "the source games", Secret: "smb-password-games", What: "the password of the source games"}

// toolJudge is a judge of one running process whose tool check answers what *answer holds.
func toolJudge(t *testing.T, answer *[]byte) (*Judge, *clock) {
	t.Helper()
	c := &clock{now: t0}
	rt := &fakeRuntime{containers: map[string]Observed{"watch": running("c1", 0, t0.Format(time.RFC3339Nano))}}
	j := aJudge(t, rt, c, filepath.Join(t.TempDir(), FileName))
	j.Probes = &Probes{AskTool: func(context.Context, string, string) (health.Answer, error) {
		return health.Read(*answer)
	}}
	check := &declaration.Health{Kind: declaration.HealthTool, Tool: "mounts_health", Interval: "30s", Timeout: "5s",
		Looks: 2, Grace: "1m30s"}
	j.Set([]Resource{{Module: "mounts", ID: "mounts.watch", Kind: KindContainer, Target: "watch", Check: check}})
	j.Look(t.Context())
	return j, c
}

func answerOf(t *testing.T, a health.Answer) []byte {
	t.Helper()
	b, err := json.Marshal(a)
	if err != nil {
		t.Fatal(err)
	}
	return b
}

func next(t *testing.T, j *Judge, c *clock, by time.Duration) State {
	t.Helper()
	c.now = c.now.Add(by)
	lookOnce(t, j)
	st, _ := j.Look(t.Context())
	return stateOf(t, st, "mounts.watch")
}

func TestAToolThatWaitsIsStatedWaitingWithItsWaitsInsideTheGrace(t *testing.T) {
	answer := answerOf(t, health.Waiting("the source games waits for its password", password))
	j, c := toolJudge(t, &answer)
	s := next(t, j, c, 0)
	if s.State != Waiting || len(s.Waits) != 1 || s.Waits[0].Secret != "smb-password-games" ||
		!strings.Contains(s.Reason, "waits for the operator: the source games waits for its password") {
		t.Fatalf("in its grace, a tool that waits: %+v", s)
	}
	if s.Streak != 0 {
		t.Fatalf("a wait counted as a failing streak: %+v", s)
	}
}

func TestAToolThatWaitsStaysWaitingAfterTheGraceAndIsNoFailingLook(t *testing.T) {
	answer := answerOf(t, health.Waiting("waits", password))
	j, c := toolJudge(t, &answer)
	for i := 0; i < 5; i++ {
		if s := next(t, j, c, 30*time.Second); s.State != Waiting {
			t.Fatalf("look %d after the grace: %+v", i, s)
		}
	}
}

func TestTheNextHealthyAnswerAfterAWaitIsHealthy(t *testing.T) {
	answer := answerOf(t, health.Waiting("waits", password))
	j, c := toolJudge(t, &answer)
	next(t, j, c, 2*time.Minute)
	answer = answerOf(t, health.Healthy("every source armed"))
	if s := next(t, j, c, 30*time.Second); s.State != Healthy || len(s.Waits) != 0 {
		t.Fatalf("healthy after a wait: %+v", s)
	}
}

func TestAnUnhealthyAnswerAfterAWaitFollowsTheFailingLooks(t *testing.T) {
	answer := answerOf(t, health.Waiting("waits", password))
	j, c := toolJudge(t, &answer)
	next(t, j, c, 2*time.Minute)
	answer = answerOf(t, health.Unhealthy("the share is unreachable"))
	s := next(t, j, c, 30*time.Second)
	if s.State == Unhealthy || s.State == Waiting || len(s.Waits) != 0 {
		t.Fatalf("one failing look after a wait made it %+v", s)
	}
	if s = next(t, j, c, 30*time.Second); s.State != Unhealthy || !strings.Contains(s.Reason, "the share is unreachable") {
		t.Fatalf("two failing looks after a wait: %+v", s)
	}
}

func TestAnAnswerWithoutStateIsJudgedAsBefore(t *testing.T) {
	answer := []byte(`{"healthy":false,"why":"down"}`)
	j, c := toolJudge(t, &answer)
	next(t, j, c, 2*time.Minute)
	if s := next(t, j, c, 30*time.Second); s.State != Unhealthy || len(s.Waits) != 0 {
		t.Fatalf("an old unhealthy answer: %+v", s)
	}
	answer = []byte(`{"healthy":true,"why":"ok"}`)
	if s := next(t, j, c, 30*time.Second); s.State != Healthy {
		t.Fatalf("an old healthy answer: %+v", s)
	}
}

func TestAMalformedWaitIsUnhealthyAndExcusesNothing(t *testing.T) {
	answer := []byte(`{"state":"waits","healthy":false,"why":"it waits","waits":[{"part":"p","what":"the password"}]}`)
	j, c := toolJudge(t, &answer)
	next(t, j, c, 2*time.Minute)
	s := next(t, j, c, 30*time.Second)
	if s.State != Unhealthy || len(s.Waits) != 0 || !strings.Contains(s.Reason, "neither a secret nor a setting") {
		t.Fatalf("a wait naming nothing: %+v", s)
	}
}