State a tool check that waits for the operator as waiting (issue 386, ADR 0283)

A module whose only fault was a secret the operator had not given yet was
stated unhealthy, failed its first-node gate and was put back. The engine
now reads the tool's answer through mesh-sdk go/health and states such a
resource waiting, with what it waits for, on one look; a malformed wait
stays unhealthy.

internal/liveness/liveness.go

@@ -39,6 +39,7 @@ import (
 	"sync"
 	"time"
 
+	"git.novox.be/novox/mesh-sdk/go/health"
 	"github.com/novox/mesh-host/internal/declaration"
 )
 
@@ -66,6 +67,9 @@ const (
 	Starting  = "starting"
 	Held      = "held"
 	Unknown   = "unknown"
+	// Waiting is a resource whose tool check says it waits for the operator, and nothing else is wrong
+	// (novox/hq ADR 0283): a statement, not a fault, said with what it waits for.
+	Waiting = "waiting"
 
 	ReasonDown       = "down"
 	ReasonRestarting = "restarting"
@@ -175,6 +179,8 @@ type State struct {
 	Since    time.Time `json:"since"`
 	Streak   int       `json:"streak,omitempty"`
 	Restarts int       `json:"restarts,omitempty"`
+	// Waits is what a resource stated Waiting waits for (ADR 0283).
+	Waits []health.Wait `json:"waits,omitempty"`
 }
 
 // CheckOf and NeedsOf are a stated resource's declared check, in a word, and the provision it exercises.
@@ -235,6 +241,9 @@ type kept struct {
 	Passed  bool   `json:"passed,omitempty"`
 	Failing int    `json:"failing,omitempty"`
 	Why     string `json:"why,omitempty"`
+	// Waits is what the last look of a tool check said the resource waits for (ADR 0283), nil when it said
+	// healthy or unhealthy; Why then is the answer's own words.
+	Waits []health.Wait `json:"waits,omitempty"`
 	// probing says a look of the engine's own is under way; due when the next is.
 	probing bool
 	due     time.Time
@@ -319,7 +328,7 @@ func (j *Judge) Set(resources []Resource) {
 		} else if ok && !sameCheck(k.Check, r.Check) {
 			// Its check changed — another kind, another port, another timing: what the old one found
 			// says nothing about the new, which looks again at once.
-			k.Check, k.Passed, k.Failing, k.Why, k.due = r.Check, false, 0, "", time.Time{}
+			k.Check, k.Passed, k.Failing, k.Why, k.Waits, k.due = r.Check, false, 0, "", nil, time.Time{}
 		}
 	}
 	for id := range j.f.Kept {
@@ -358,7 +367,7 @@ func (j *Judge) Look(ctx context.Context) (Statement, bool) {
 		j.judge(k, o, now, r.Kind == KindContainer && held[r.Target], blind, why)
 		seen[r.ID] = true
 		st.Resources = append(st.Resources, State{Resource: k.Resource, State: k.State, Reason: k.Reason, Since: k.Since,
-			Streak: k.Streak, Restarts: k.Counted})
+			Streak: k.Streak, Restarts: k.Counted, Waits: waitsOf(k)})
 		word := k.State + "/" + k.Reason
 		if j.said[r.ID] != word {
 			changed = true
@@ -392,7 +401,7 @@ func (j *Judge) judge(k *kept, o Observed, now time.Time, held, blind bool, why
 		k.Seen, k.Identity, k.RuntimeRestarts, k.RuntimeStarted = o.Found, o.Identity, o.Restarts, o.Started
 		k.Started, k.Recent = started, nil
 		// Every start is judged ready afresh (to-be 48 §4).
-		k.Passed, k.Failing, k.Why, k.due = false, 0, "", time.Time{}
+		k.Passed, k.Failing, k.Why, k.Waits, k.due = false, 0, "", nil, time.Time{}
 	}
 
 	// **Held is neither alive nor dead**, and the window ending is a start: judged from a fresh grace.
@@ -562,3 +571,11 @@ func firstLine(s string) string {
 	line, _, _ := strings.Cut(strings.TrimSpace(s), "\n")
 	return line
 }
+
+// waitsOf is what a resource stated Waiting waits for, and nothing for any other state.
+func waitsOf(k *kept) []health.Wait {
+	if k.State != Waiting {
+		return nil
+	}
+	return append([]health.Wait(nil), k.Waits...)
+}

internal/liveness/readiness.go

@@ -14,6 +14,7 @@ import (
 	"strings"
 	"time"
 
+	"git.novox.be/novox/mesh-sdk/go/health"
 	"github.com/novox/mesh-host/internal/declaration"
 )
 
@@ -77,6 +78,10 @@ func ready(k *kept, o Observed, inGrace bool) (string, string) {
 		}
 	}
 	switch {
+	case len(k.Waits) > 0:
+		// **A wait for the operator is a statement, not a flake** (novox/hq ADR 0283): one look that says it
+		// is enough, inside the grace too; the next look that says anything else replaces it.
+		return Waiting, "waits for the operator: " + orSaid(k.Why, "it says what it waits for")
 	case !inGrace && k.Failing >= c.Looks:
 		return Unhealthy, c.Words() + ": " + orSaid(k.Why, "it failed")
 	case k.Passed:
@@ -156,8 +161,8 @@ func (j *Judge) Probe(ctx context.Context) {
 			}
 			go func(d dueLook) {
 				defer func() { <-slots }()
-				ok, why := j.probes().Look(ctx, d.module, d.check)
-				j.Record(d.id, d.started, ok, why)
+				ok, why, waits := j.probes().Look(ctx, d.module, d.check)
+				j.Record(d.id, d.started, ok, why, waits...)
 			}(d)
 		}
 	}
@@ -190,9 +195,10 @@ func (j *Judge) due() []dueLook {
 }
 
 // Record keeps what one of the engine's own looks found, for the next look to fold in: a pass is a pass
-// whenever it came; a failure counts only after the grace. A look begun before the resource's current
-// start says nothing about it.
-func (j *Judge) Record(id string, started time.Time, ok bool, why string) {
+// whenever it came; a failure counts only after the grace; a look that says the resource waits for the
+// operator (waits, ADR 0283) is kept as that, in the grace too, and counts as no failing look. A look begun
+// before the resource's current start says nothing about it.
+func (j *Judge) Record(id string, started time.Time, ok bool, why string, waits ...health.Wait) {
 	j.mu.Lock()
 	defer j.mu.Unlock()
 	k := j.f.Kept[id]
@@ -203,9 +209,12 @@ func (j *Judge) Record(id string, started time.Time, ok bool, why string) {
 	if !k.Started.Equal(started) || k.Check == nil {
 		return
 	}
+	k.Waits = nil
 	switch {
 	case ok:
 		k.Passed, k.Failing, k.Why = true, 0, ""
+	case len(waits) > 0:
+		k.Failing, k.Why, k.Waits = 0, why, append([]health.Wait(nil), waits...)
 	case j.Now().Before(k.Started.Add(j.graceOf(k.Resource))):
 		k.Why = why
 	default:
@@ -227,11 +236,12 @@ type Probes struct {
 	// Host is where an endpoint's port is dialled: this machine, by default its loopback.
 	Host string
 	// AskTool asks a module's tool on this machine's own node tools; nil says it cannot be asked.
-	AskTool func(ctx context.Context, module, tool string) (healthy bool, why string, err error)
+	AskTool func(ctx context.Context, module, tool string) (health.Answer, error)
 }
 
-// Look makes one look of a check, and says whether it passed and, when not, what it found.
-func (p *Probes) Look(ctx context.Context, module string, c *declaration.Health) (bool, string) {
+// Look makes one look of a check, and says whether it passed and, when not, what it found — and, for a
+// tool that answers it waits for the operator (ADR 0283), what it waits for. Only a tool check can wait.
+func (p *Probes) Look(ctx context.Context, module string, c *declaration.Health) (bool, string, []health.Wait) {
 	timeout := c.Within()
 	if timeout <= 0 {
 		timeout = 5 * time.Second
@@ -240,30 +250,33 @@ func (p *Probes) Look(ctx context.Context, module string, c *declaration.Health)
 	defer cancel()
 	switch c.Kind {
 	case declaration.HealthHTTP:
-		return p.http(ctx, c, timeout)
+		ok, why := p.http(ctx, c, timeout)
+		return ok, why, nil
 	case declaration.HealthTCP:
 		conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", p.address(c.Port))
 		if err != nil {
-			return false, dialWords(err, timeout)
+			return false, dialWords(err, timeout), nil
 		}
 		_ = conn.Close()
-		return true, ""
+		return true, "", nil
 	case declaration.HealthTool:
 		if p.AskTool == nil {
-			return false, "its tool cannot be asked: this engine has no link to the node tools"
+			return false, "its tool cannot be asked: this engine has no link to the node tools", nil
 		}
-		healthy, why, err := p.AskTool(ctx, module, c.Tool)
+		a, err := p.AskTool(ctx, module, c.Tool)
 		switch {
 		case errors.Is(err, context.DeadlineExceeded):
-			return false, fmt.Sprintf("no answer within %s", timeout)
+			return false, fmt.Sprintf("no answer within %s", timeout), nil
 		case err != nil:
-			return false, firstLine(err.Error())
-		case !healthy:
-			return false, orSaid(why, "it answered not healthy, and not why")
+			return false, firstLine(err.Error()), nil
+		case a.State == health.StateWaits && len(a.Waits) > 0:
+			return false, orSaid(a.Why, "it waits for the operator, and not why"), a.Waits
+		case !a.Healthy:
+			return false, orSaid(a.Why, "it answered not healthy, and not why"), nil
 		}
-		return true, ""
+		return true, "", nil
 	}
-	return false, "the engine does not make a " + c.Kind + " look itself"
+	return false, "the engine does not make a " + c.Kind + " look itself", nil
 }
 
 func (p *Probes) address(port int) string {

internal/liveness/readiness_test.go

@@ -13,6 +13,7 @@ import (
 	"testing"
 	"time"
 
+	"git.novox.be/novox/mesh-sdk/go/health"
 	"github.com/novox/mesh-host/internal/declaration"
 )
 
@@ -40,8 +41,8 @@ func portOf(t *testing.T, url string) int {
 func lookOnce(t *testing.T, j *Judge) {
 	t.Helper()
 	for _, d := range j.due() {
-		ok, why := j.probes().Look(t.Context(), d.module, d.check)
-		j.Record(d.id, d.started, ok, why)
+		ok, why, waits := j.probes().Look(t.Context(), d.module, d.check)
+		j.Record(d.id, d.started, ok, why, waits...)
 	}
 }
 
@@ -182,11 +183,11 @@ func TestNeverMoreLooksThanTheBudget(t *testing.T) {
 
 func TestAToolCheckIsAskedOfTheNodeTools(t *testing.T) {
 	asked := ""
-	p := &Probes{AskTool: func(_ context.Context, module, tool string) (bool, string, error) {
+	p := &Probes{AskTool: func(_ context.Context, module, tool string) (health.Answer, error) {
 		asked = module + "." + tool
-		return false, "the admin refused the minted secret", nil
+		return health.Unhealthy("the admin refused the minted secret"), nil
 	}}
-	ok, why := p.Look(t.Context(), "keycloak", &declaration.Health{Kind: declaration.HealthTool, Tool: "keycloak_admin_health",
+	ok, why, _ := p.Look(t.Context(), "keycloak", &declaration.Health{Kind: declaration.HealthTool, Tool: "keycloak_admin_health",
 		Interval: "30s", Timeout: "5s", Looks: 2, Grace: "0s"})
 	if ok || asked != "keycloak.keycloak_admin_health" || !strings.Contains(why, "refused the minted secret") {
 		t.Fatalf("asked %q: %v %q", asked, ok, why)
@@ -198,16 +199,16 @@ func TestAToolCheckIsAskedOfTheNodeTools(t *testing.T) {
 func TestAToolCheckThatGetsNoAnswerSaysHowLongItWaited(t *testing.T) {
 	check := &declaration.Health{Kind: declaration.HealthTool, Tool: "sensors_health",
 		Interval: "30s", Timeout: "5s", Looks: 2, Grace: "0s"}
-	p := &Probes{AskTool: func(context.Context, string, string) (bool, string, error) {
-		return false, "", fmt.Errorf("no answer from mesh.mod.sensors.tool.sensors_health.anchor: %w", context.DeadlineExceeded)
+	p := &Probes{AskTool: func(context.Context, string, string) (health.Answer, error) {
+		return health.Answer{}, fmt.Errorf("no answer from mesh.mod.sensors.tool.sensors_health.anchor: %w", context.DeadlineExceeded)
 	}}
-	if ok, why := p.Look(t.Context(), "sensors", check); ok || why != "no answer within 5s" {
+	if ok, why, _ := p.Look(t.Context(), "sensors", check); ok || why != "no answer within 5s" {
 		t.Fatalf("%v %q", ok, why)
 	}
-	p.AskTool = func(context.Context, string, string) (bool, string, error) {
-		return false, "", errors.New("no link to the bus is open")
+	p.AskTool = func(context.Context, string, string) (health.Answer, error) {
+		return health.Answer{}, errors.New("no link to the bus is open")
 	}
-	if ok, why := p.Look(t.Context(), "sensors", check); ok || why != "no link to the bus is open" {
+	if ok, why, _ := p.Look(t.Context(), "sensors", check); ok || why != "no link to the bus is open" {
 		t.Fatalf("%v %q", ok, why)
 	}
 }

internal/liveness/replay_test.go

@@ -168,8 +168,8 @@ func TestReplaySilentWebAppIsSaidUnhealthy(t *testing.T) {
 	looks := 0
 	for time.Since(began) < 2*time.Minute {
 		for _, d := range j.due() {
-			ok, why := j.probes().Look(t.Context(), d.module, d.check)
-			j.Record(d.id, d.started, ok, why)
+			ok, why, waits := j.probes().Look(t.Context(), d.module, d.check)
+			j.Record(d.id, d.started, ok, why, waits...)
 			looks++
 		}
 		st, _ = j.Look(t.Context())
@@ -183,7 +183,7 @@ func TestReplaySilentWebAppIsSaidUnhealthy(t *testing.T) {
 		t.Fatalf("a web application answering nothing was not said unhealthy within two looks (%d looks): %+v", looks, s)
 	}
 	// And a TCP check would have said it reachable: the port is open.
-	if ok, why := tcp.Look(t.Context(), "app", timing(&declaration.Health{Kind: declaration.HealthTCP, Port: published})); !ok {
+	if ok, why, _ := tcp.Look(t.Context(), "app", timing(&declaration.Health{Kind: declaration.HealthTCP, Port: published})); !ok {
 		t.Fatalf("a tcp check could not connect to the silent application: %s", why)
 	}
 	t.Logf("said %s (%s) after %d looks, %s after the judging began; a TCP check connects", s.State, s.Reason, looks,

internal/liveness/waiting_test.go

@@ -0,0 +1,122 @@
+package liveness
+
+import (
+	"context"
+	"encoding/json"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/health"
+	"github.com/novox/mesh-host/internal/declaration"
+)
+
+// A module's tool check may say it waits for the operator (novox/hq ADR 0283, issue 386): a named secret or
+// setting not given yet, and nothing else wrong. The engine states that resource waiting, with its waits, on
+// one look, inside the grace too; the next look that says anything else replaces it.
+
+var password = health.Wait{Part: "the source games", Secret: "smb-password-games", What: "the password of the source games"}
+
+// toolJudge is a judge of one running process whose tool check answers what *answer holds.
+func toolJudge(t *testing.T, answer *[]byte) (*Judge, *clock) {
+	t.Helper()
+	c := &clock{now: t0}
+	rt := &fakeRuntime{containers: map[string]Observed{"watch": running("c1", 0, t0.Format(time.RFC3339Nano))}}
+	j := aJudge(t, rt, c, filepath.Join(t.TempDir(), FileName))
+	j.Probes = &Probes{AskTool: func(context.Context, string, string) (health.Answer, error) {
+		return health.Read(*answer)
+	}}
+	check := &declaration.Health{Kind: declaration.HealthTool, Tool: "mounts_health", Interval: "30s", Timeout: "5s",
+		Looks: 2, Grace: "1m30s"}
+	j.Set([]Resource{{Module: "mounts", ID: "mounts.watch", Kind: KindContainer, Target: "watch", Check: check}})
+	j.Look(t.Context())
+	return j, c
+}
+
+func answerOf(t *testing.T, a health.Answer) []byte {
+	t.Helper()
+	b, err := json.Marshal(a)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return b
+}
+
+func next(t *testing.T, j *Judge, c *clock, by time.Duration) State {
+	t.Helper()
+	c.now = c.now.Add(by)
+	lookOnce(t, j)
+	st, _ := j.Look(t.Context())
+	return stateOf(t, st, "mounts.watch")
+}
+
+func TestAToolThatWaitsIsStatedWaitingWithItsWaitsInsideTheGrace(t *testing.T) {
+	answer := answerOf(t, health.Waiting("the source games waits for its password", password))
+	j, c := toolJudge(t, &answer)
+	s := next(t, j, c, 0)
+	if s.State != Waiting || len(s.Waits) != 1 || s.Waits[0].Secret != "smb-password-games" ||
+		!strings.Contains(s.Reason, "waits for the operator: the source games waits for its password") {
+		t.Fatalf("in its grace, a tool that waits: %+v", s)
+	}
+	if s.Streak != 0 {
+		t.Fatalf("a wait counted as a failing streak: %+v", s)
+	}
+}
+
+func TestAToolThatWaitsStaysWaitingAfterTheGraceAndIsNoFailingLook(t *testing.T) {
+	answer := answerOf(t, health.Waiting("waits", password))
+	j, c := toolJudge(t, &answer)
+	for i := 0; i < 5; i++ {
+		if s := next(t, j, c, 30*time.Second); s.State != Waiting {
+			t.Fatalf("look %d after the grace: %+v", i, s)
+		}
+	}
+}
+
+func TestTheNextHealthyAnswerAfterAWaitIsHealthy(t *testing.T) {
+	answer := answerOf(t, health.Waiting("waits", password))
+	j, c := toolJudge(t, &answer)
+	next(t, j, c, 2*time.Minute)
+	answer = answerOf(t, health.Healthy("every source armed"))
+	if s := next(t, j, c, 30*time.Second); s.State != Healthy || len(s.Waits) != 0 {
+		t.Fatalf("healthy after a wait: %+v", s)
+	}
+}
+
+func TestAnUnhealthyAnswerAfterAWaitFollowsTheFailingLooks(t *testing.T) {
+	answer := answerOf(t, health.Waiting("waits", password))
+	j, c := toolJudge(t, &answer)
+	next(t, j, c, 2*time.Minute)
+	answer = answerOf(t, health.Unhealthy("the share is unreachable"))
+	s := next(t, j, c, 30*time.Second)
+	if s.State == Unhealthy || s.State == Waiting || len(s.Waits) != 0 {
+		t.Fatalf("one failing look after a wait made it %+v", s)
+	}
+	if s = next(t, j, c, 30*time.Second); s.State != Unhealthy || !strings.Contains(s.Reason, "the share is unreachable") {
+		t.Fatalf("two failing looks after a wait: %+v", s)
+	}
+}
+
+func TestAnAnswerWithoutStateIsJudgedAsBefore(t *testing.T) {
+	answer := []byte(`{"healthy":false,"why":"down"}`)
+	j, c := toolJudge(t, &answer)
+	next(t, j, c, 2*time.Minute)
+	if s := next(t, j, c, 30*time.Second); s.State != Unhealthy || len(s.Waits) != 0 {
+		t.Fatalf("an old unhealthy answer: %+v", s)
+	}
+	answer = []byte(`{"healthy":true,"why":"ok"}`)
+	if s := next(t, j, c, 30*time.Second); s.State != Healthy {
+		t.Fatalf("an old healthy answer: %+v", s)
+	}
+}
+
+func TestAMalformedWaitIsUnhealthyAndExcusesNothing(t *testing.T) {
+	answer := []byte(`{"state":"waits","healthy":false,"why":"it waits","waits":[{"part":"p","what":"the password"}]}`)
+	j, c := toolJudge(t, &answer)
+	next(t, j, c, 2*time.Minute)
+	s := next(t, j, c, 30*time.Second)
+	if s.State != Unhealthy || len(s.Waits) != 0 || !strings.Contains(s.Reason, "neither a secret nor a setting") {
+		t.Fatalf("a wait naming nothing: %+v", s)
+	}
+}