State a tool check that waits for the operator as waiting (issue 386, ADR 0283)

A module whose only fault was a secret the operator had not given yet was
stated unhealthy, failed its first-node gate and was put back. The engine
now reads the tool's answer through mesh-sdk go/health and states such a
resource waiting, with what it waits for, on one look; a malformed wait
stays unhealthy.

internal/liveness/waiting_test.go

@@ -0,0 +1,122 @@
+package liveness
+
+import (
+	"context"
+	"encoding/json"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/health"
+	"github.com/novox/mesh-host/internal/declaration"
+)
+
+// A module's tool check may say it waits for the operator (novox/hq ADR 0283, issue 386): a named secret or
+// setting not given yet, and nothing else wrong. The engine states that resource waiting, with its waits, on
+// one look, inside the grace too; the next look that says anything else replaces it.
+
+var password = health.Wait{Part: "the source games", Secret: "smb-password-games", What: "the password of the source games"}
+
+// toolJudge is a judge of one running process whose tool check answers what *answer holds.
+func toolJudge(t *testing.T, answer *[]byte) (*Judge, *clock) {
+	t.Helper()
+	c := &clock{now: t0}
+	rt := &fakeRuntime{containers: map[string]Observed{"watch": running("c1", 0, t0.Format(time.RFC3339Nano))}}
+	j := aJudge(t, rt, c, filepath.Join(t.TempDir(), FileName))
+	j.Probes = &Probes{AskTool: func(context.Context, string, string) (health.Answer, error) {
+		return health.Read(*answer)
+	}}
+	check := &declaration.Health{Kind: declaration.HealthTool, Tool: "mounts_health", Interval: "30s", Timeout: "5s",
+		Looks: 2, Grace: "1m30s"}
+	j.Set([]Resource{{Module: "mounts", ID: "mounts.watch", Kind: KindContainer, Target: "watch", Check: check}})
+	j.Look(t.Context())
+	return j, c
+}
+
+func answerOf(t *testing.T, a health.Answer) []byte {
+	t.Helper()
+	b, err := json.Marshal(a)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return b
+}
+
+func next(t *testing.T, j *Judge, c *clock, by time.Duration) State {
+	t.Helper()
+	c.now = c.now.Add(by)
+	lookOnce(t, j)
+	st, _ := j.Look(t.Context())
+	return stateOf(t, st, "mounts.watch")
+}
+
+func TestAToolThatWaitsIsStatedWaitingWithItsWaitsInsideTheGrace(t *testing.T) {
+	answer := answerOf(t, health.Waiting("the source games waits for its password", password))
+	j, c := toolJudge(t, &answer)
+	s := next(t, j, c, 0)
+	if s.State != Waiting || len(s.Waits) != 1 || s.Waits[0].Secret != "smb-password-games" ||
+		!strings.Contains(s.Reason, "waits for the operator: the source games waits for its password") {
+		t.Fatalf("in its grace, a tool that waits: %+v", s)
+	}
+	if s.Streak != 0 {
+		t.Fatalf("a wait counted as a failing streak: %+v", s)
+	}
+}
+
+func TestAToolThatWaitsStaysWaitingAfterTheGraceAndIsNoFailingLook(t *testing.T) {
+	answer := answerOf(t, health.Waiting("waits", password))
+	j, c := toolJudge(t, &answer)
+	for i := 0; i < 5; i++ {
+		if s := next(t, j, c, 30*time.Second); s.State != Waiting {
+			t.Fatalf("look %d after the grace: %+v", i, s)
+		}
+	}
+}
+
+func TestTheNextHealthyAnswerAfterAWaitIsHealthy(t *testing.T) {
+	answer := answerOf(t, health.Waiting("waits", password))
+	j, c := toolJudge(t, &answer)
+	next(t, j, c, 2*time.Minute)
+	answer = answerOf(t, health.Healthy("every source armed"))
+	if s := next(t, j, c, 30*time.Second); s.State != Healthy || len(s.Waits) != 0 {
+		t.Fatalf("healthy after a wait: %+v", s)
+	}
+}
+
+func TestAnUnhealthyAnswerAfterAWaitFollowsTheFailingLooks(t *testing.T) {
+	answer := answerOf(t, health.Waiting("waits", password))
+	j, c := toolJudge(t, &answer)
+	next(t, j, c, 2*time.Minute)
+	answer = answerOf(t, health.Unhealthy("the share is unreachable"))
+	s := next(t, j, c, 30*time.Second)
+	if s.State == Unhealthy || s.State == Waiting || len(s.Waits) != 0 {
+		t.Fatalf("one failing look after a wait made it %+v", s)
+	}
+	if s = next(t, j, c, 30*time.Second); s.State != Unhealthy || !strings.Contains(s.Reason, "the share is unreachable") {
+		t.Fatalf("two failing looks after a wait: %+v", s)
+	}
+}
+
+func TestAnAnswerWithoutStateIsJudgedAsBefore(t *testing.T) {
+	answer := []byte(`{"healthy":false,"why":"down"}`)
+	j, c := toolJudge(t, &answer)
+	next(t, j, c, 2*time.Minute)
+	if s := next(t, j, c, 30*time.Second); s.State != Unhealthy || len(s.Waits) != 0 {
+		t.Fatalf("an old unhealthy answer: %+v", s)
+	}
+	answer = []byte(`{"healthy":true,"why":"ok"}`)
+	if s := next(t, j, c, 30*time.Second); s.State != Healthy {
+		t.Fatalf("an old healthy answer: %+v", s)
+	}
+}
+
+func TestAMalformedWaitIsUnhealthyAndExcusesNothing(t *testing.T) {
+	answer := []byte(`{"state":"waits","healthy":false,"why":"it waits","waits":[{"part":"p","what":"the password"}]}`)
+	j, c := toolJudge(t, &answer)
+	next(t, j, c, 2*time.Minute)
+	s := next(t, j, c, 30*time.Second)
+	if s.State != Unhealthy || len(s.Waits) != 0 || !strings.Contains(s.Reason, "neither a secret nor a setting") {
+		t.Fatalf("a wait naming nothing: %+v", s)
+	}
+}