State a tool check that waits for the operator as waiting (issue 386, ADR 0283)

A module whose only fault was a secret the operator had not given yet was
stated unhealthy, failed its first-node gate and was put back. The engine
now reads the tool's answer through mesh-sdk go/health and states such a
resource waiting, with what it waits for, on one look; a malformed wait
stays unhealthy.

cmd/mesh-host/main.go

@@ -1619,7 +1619,8 @@ func healthAsReported(st liveness.Statement, ns *network.Statement) *link.Health
 	for _, r := range st.Resources {
 		h.Resources = append(h.Resources, link.ResourceHealth{Module: r.Module, Resource: r.ID, Kind: r.Kind,
 			Target: r.Target, State: r.State, Reason: r.Reason, Since: r.Since.UTC(), Streak: r.Streak,
-			Restarts: r.Restarts, Check: r.CheckOf(), Needs: r.NeedsOf(), Account: userOf(r.Scope, r.User)})
+			Restarts: r.Restarts, Check: r.CheckOf(), Needs: r.NeedsOf(), Account: userOf(r.Scope, r.User),
+			Waits: r.Waits})
 	}
 	if ns != nil && ns.State != "" {
 		h.Network = &link.NetworkHealth{State: ns.State, Since: ns.Since.UTC(), Parts: []link.NetworkPart{}}

go.mod

@@ -3,7 +3,7 @@ module github.com/novox/mesh-host
 go 1.26.0
 
 require (
-	git.novox.be/novox/mesh-sdk/go v0.1.11-0.20261009143344-f047d0a4a970
+	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c
 	github.com/nats-io/nats.go v1.54.0
 	golang.org/x/crypto v0.57.0
 	golang.org/x/sys v0.48.0

go.sum

@@ -1,5 +1,9 @@
 git.novox.be/novox/mesh-sdk/go v0.1.11-0.20261009143344-f047d0a4a970 h1:9tFDQsgmI+4X7/BpZGXIr+HemPKE7YddYGqWV0lINAI=
 git.novox.be/novox/mesh-sdk/go v0.1.11-0.20261009143344-f047d0a4a970/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010190644-622b6b4f556d h1:BR8uwv4BqyLaBwITuLylrd9DRUnu0D5DjJw0RuQSXbk=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010190644-622b6b4f556d/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c h1:l5onJwoIeH8yE/PjVlEeoPyXBsHp2NQiWLllz2fwX7s=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
 github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
 github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
 github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=

internal/link/asktool_nats_test.go

@@ -29,11 +29,11 @@ func TestNatsAToolCheckIsAskedOnTheLink(t *testing.T) {
 	q.attach(t.Context(), &natsLink{conn: conn, js: js, node: node})
 	ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second)
 	defer cancel()
-	healthy, why, err := q.AskTool(ctx, "sensors", "sensors_health")
+	a, err := q.AskTool(ctx, "sensors", "sensors_health")
-	if err != nil || !healthy {
+	if err != nil || !a.Healthy {
-		t.Fatalf("a served healthy tool read as %v %q (%v)", healthy, why, err)
+		t.Fatalf("a served healthy tool read as %+v (%v)", a, err)
 	}
-	_, _, err = q.AskTool(ctx, "sensors", "nothing_serves_this")
+	_, err = q.AskTool(ctx, "sensors", "nothing_serves_this")
 	if err == nil || !strings.Contains(err.Error(), "nothing on this machine answers") {
 		t.Fatalf("a tool nothing serves read as %v", err)
 	}

internal/link/health_test.go

@@ -62,8 +62,8 @@ func TestAToolsAnswerIsHealthyOnlyWhenItSaysSo(t *testing.T) {
 		`{"error":"503 no responders"}`:                                     false,
 		`{"result":"yes"}`:                                                  false,
 	} {
-		got, _, err := ReadToolAnswer([]byte(reply))
+		got, err := ReadToolAnswer([]byte(reply))
-		if err != nil || got != want {
+		if err != nil || got.Healthy != want {
 			t.Errorf("%s read as %v (%v)", reply, got, err)
 		}
 	}
@@ -78,13 +78,13 @@ func TestAToolsAnswerIsHealthyOnlyWhenItSaysSo(t *testing.T) {
 // ask, the check fails and says which.
 func TestAToolCheckIsAskedOnTheLinkTheEngineHolds(t *testing.T) {
 	q := &Queue{Membership: Membership{Node: "anchor"}}
-	if _, _, err := q.AskTool(t.Context(), "sensors", "sensors_health"); err == nil || err.Error() != "no link to the bus is open" {
+	if _, err := q.AskTool(t.Context(), "sensors", "sensors_health"); err == nil || err.Error() != "no link to the bus is open" {
 		t.Fatalf("with no link open: %v", err)
 	}
 
 	quiet := newQuietLink()
 	q.attach(t.Context(), quiet)
-	if _, _, err := q.AskTool(t.Context(), "sensors", "sensors_health"); err == nil || err.Error() != "the link to the bus open now cannot ask a tool" {
+	if _, err := q.AskTool(t.Context(), "sensors", "sensors_health"); err == nil || err.Error() != "the link to the bus open now cannot ask a tool" {
 		t.Fatalf("on a link that cannot ask: %v", err)
 	}
 	q.detach(quiet)
@@ -93,7 +93,7 @@ func TestAToolCheckIsAskedOnTheLinkTheEngineHolds(t *testing.T) {
 	// there), never stop at the queue's assertion.
 	held := &natsLink{node: "anchor"}
 	q.attach(t.Context(), held)
-	_, _, err := q.AskTool(t.Context(), "sensors", "sensors_health")
+	_, err := q.AskTool(t.Context(), "sensors", "sensors_health")
 	if err == nil {
 		t.Fatal("asked on no connection and was answered")
 	}
@@ -101,3 +101,18 @@ func TestAToolCheckIsAskedOnTheLinkTheEngineHolds(t *testing.T) {
 		t.Fatalf("the NATS link was not asked: %v", err)
 	}
 }
+
+// A tool may answer that it waits for the operator (novox/hq ADR 0283): read with what it waits for, and not
+// healthy, so nothing reading `healthy` alone takes it for a pass.
+func TestAToolsWaitIsReadWithWhatItWaitsFor(t *testing.T) {
+	got, err := ReadToolAnswer([]byte(`{"result":{"state":"waits","healthy":false,"why":"the source games waits",` +
+		`"waits":[{"part":"the source games","secret":"smb-password-games","what":"the password of the source games"}]}}`))
+	if err != nil || got.Healthy || got.State != "waits" || len(got.Waits) != 1 || got.Waits[0].Secret != "smb-password-games" {
+		t.Fatalf("read as %+v (%v)", got, err)
+	}
+	h := ResourceHealth{Module: "mounts", Resource: "mounts.watch", State: StateWaiting, Waits: got.Waits}
+	b, _ := json.Marshal(h)
+	if !strings.Contains(string(b), `"state":"waiting"`) || !strings.Contains(string(b), `"waits":[{"part":"the source games","secret":"smb-password-games"`) {
+		t.Fatalf("said on the wire as %s", b)
+	}
+}

internal/link/messages.go

@@ -4,6 +4,8 @@ import (
 	"strconv"
 	"strings"
 	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/health"
 )
 
 // The wire formats shared with the control plane, which defines them separately because this
@@ -335,6 +337,10 @@ const (
 	StateHeld = "held"
 	// StateUnknown is a resource whose state could not be read.
 	StateUnknown = "unknown"
+	// StateWaiting is a resource whose tool check says it waits for the operator — a named own secret or
+	// setting of the module not given yet — and nothing else is wrong (novox/hq ADR 0283). Not a fault: the
+	// controller checks the wait, and an older controller reads it as not healthy and raises nothing.
+	StateWaiting = "waiting"
 )
 
 // The reasons an unhealthy resource is said with, when it is liveness that judged it.
@@ -376,6 +382,9 @@ type ResourceHealth struct {
 	// Search is how far the setuid search got, on an account's verdict that waits for it (novox/hq issue
 	// 361): Reason says the same in words. Absent otherwise.
 	Search *SetuidSearch `json:"search,omitempty"`
+	// Waits is what a resource stated waiting waits for (novox/hq ADR 0283): each one own secret or one
+	// setting of the module by name, the part, and what the operator gives, in words. Absent otherwise.
+	Waits []health.Wait `json:"waits,omitempty"`
 }
 
 // SetuidSearch is how far a running search for setuid programs got, as of its last checkpoint. Counts, times

internal/link/queue.go

@@ -9,6 +9,8 @@ import (
 	"fmt"
 	"sync"
 	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/health"
 )
 
 // Queue is the node-engine's one apply queue (novox/hq to-be 45 §6, ADR 0227 rule 1).
@@ -531,21 +533,21 @@ func (q *Queue) timeout() time.Duration {
 // AskTool asks a module's tool on this machine's own node tools, as a declared health check (novox/hq ADR
 // 0240, to-be 48 §3), and reads its answer: healthy or not, with why. An error when there is no link open
 // or nothing answered — said as the check's failure, never as healthy.
-func (q *Queue) AskTool(ctx context.Context, module, tool string) (bool, string, error) {
+func (q *Queue) AskTool(ctx context.Context, module, tool string) (health.Answer, error) {
 	q.init()
 	q.mu.Lock()
 	bus := q.bus
 	q.mu.Unlock()
 	if bus == nil {
-		return false, "", errors.New("no link to the bus is open")
+		return health.Answer{}, errors.New("no link to the bus is open")
 	}
 	asker, ok := bus.(ToolBus)
 	if !ok {
-		return false, "", errors.New("the link to the bus open now cannot ask a tool")
+		return health.Answer{}, errors.New("the link to the bus open now cannot ask a tool")
 	}
 	reply, err := asker.Ask(ctx, ToolSubject(module, tool, q.Membership.Node), []byte("{}"))
 	if err != nil {
-		return false, "", err
+		return health.Answer{}, err
 	}
 	return ReadToolAnswer(reply)
 }
@@ -572,24 +574,23 @@ func (q *Queue) AskController(ctx context.Context, body []byte) ([]byte, error)
 var ErrNotAsked = errors.New("not asked")
 
 // ReadToolAnswer reads a tool's reply envelope — `{result, error}`, as the node tools answer every call —
-// as a health answer: an error is not healthy, and so is a result that does not say `healthy: true`.
+// as a health answer, through the SDK's reader (novox/hq ADR 0283): healthy, unhealthy, or waits for the
-func ReadToolAnswer(reply []byte) (bool, string, error) {
+// operator with what it waits for. An error, and a result that is not an answer, are unhealthy; a wait the
+// SDK cannot check is unhealthy too, so a malformed one never excuses anything.
+func ReadToolAnswer(reply []byte) (health.Answer, error) {
 	var env struct {
 		Result json.RawMessage `json:"result"`
 		Error  string          `json:"error"`
 	}
 	if err := json.Unmarshal(reply, &env); err != nil {
-		return false, "", fmt.Errorf("its answer is not one: %w", err)
+		return health.Answer{}, fmt.Errorf("its answer is not one: %w", err)
 	}
 	if env.Error != "" {
-		return false, env.Error, nil
+		return health.Unhealthy(env.Error), nil
 	}
-	var a struct {
+	a, err := health.Read(env.Result)
-		Healthy bool   `json:"healthy"`
+	if err != nil {
-		Why     string `json:"why"`
+		return health.Unhealthy("it answered something other than healthy or not"), nil
 	}
-	if err := json.Unmarshal(env.Result, &a); err != nil {
+	return a, nil
-		return false, "it answered something other than healthy or not", nil
-	}
-	return a.Healthy, a.Why, nil
 }

internal/link/replay331_test.go

@@ -15,7 +15,7 @@ import (
 func TestReplay331(t *testing.T) {
 	q := &Queue{Membership: Membership{Node: "laptop"}}
 	q.attach(t.Context(), &natsLink{node: "laptop"})
-	_, _, err := q.AskTool(t.Context(), "sensors", "sensors_health")
+	_, err := q.AskTool(t.Context(), "sensors", "sensors_health")
 	if err == nil {
 		t.Fatal("asked on no connection and was answered")
 	}

internal/liveness/liveness.go

@@ -39,6 +39,7 @@ import (
 	"sync"
 	"time"
 
+	"git.novox.be/novox/mesh-sdk/go/health"
 	"github.com/novox/mesh-host/internal/declaration"
 )
 
@@ -66,6 +67,9 @@ const (
 	Starting  = "starting"
 	Held      = "held"
 	Unknown   = "unknown"
+	// Waiting is a resource whose tool check says it waits for the operator, and nothing else is wrong
+	// (novox/hq ADR 0283): a statement, not a fault, said with what it waits for.
+	Waiting = "waiting"
 
 	ReasonDown       = "down"
 	ReasonRestarting = "restarting"
@@ -175,6 +179,8 @@ type State struct {
 	Since    time.Time `json:"since"`
 	Streak   int       `json:"streak,omitempty"`
 	Restarts int       `json:"restarts,omitempty"`
+	// Waits is what a resource stated Waiting waits for (ADR 0283).
+	Waits []health.Wait `json:"waits,omitempty"`
 }
 
 // CheckOf and NeedsOf are a stated resource's declared check, in a word, and the provision it exercises.
@@ -235,6 +241,9 @@ type kept struct {
 	Passed  bool   `json:"passed,omitempty"`
 	Failing int    `json:"failing,omitempty"`
 	Why     string `json:"why,omitempty"`
+	// Waits is what the last look of a tool check said the resource waits for (ADR 0283), nil when it said
+	// healthy or unhealthy; Why then is the answer's own words.
+	Waits []health.Wait `json:"waits,omitempty"`
 	// probing says a look of the engine's own is under way; due when the next is.
 	probing bool
 	due     time.Time
@@ -319,7 +328,7 @@ func (j *Judge) Set(resources []Resource) {
 		} else if ok && !sameCheck(k.Check, r.Check) {
 			// Its check changed — another kind, another port, another timing: what the old one found
 			// says nothing about the new, which looks again at once.
-			k.Check, k.Passed, k.Failing, k.Why, k.due = r.Check, false, 0, "", time.Time{}
+			k.Check, k.Passed, k.Failing, k.Why, k.Waits, k.due = r.Check, false, 0, "", nil, time.Time{}
 		}
 	}
 	for id := range j.f.Kept {
@@ -358,7 +367,7 @@ func (j *Judge) Look(ctx context.Context) (Statement, bool) {
 		j.judge(k, o, now, r.Kind == KindContainer && held[r.Target], blind, why)
 		seen[r.ID] = true
 		st.Resources = append(st.Resources, State{Resource: k.Resource, State: k.State, Reason: k.Reason, Since: k.Since,
-			Streak: k.Streak, Restarts: k.Counted})
+			Streak: k.Streak, Restarts: k.Counted, Waits: waitsOf(k)})
 		word := k.State + "/" + k.Reason
 		if j.said[r.ID] != word {
 			changed = true
@@ -392,7 +401,7 @@ func (j *Judge) judge(k *kept, o Observed, now time.Time, held, blind bool, why
 		k.Seen, k.Identity, k.RuntimeRestarts, k.RuntimeStarted = o.Found, o.Identity, o.Restarts, o.Started
 		k.Started, k.Recent = started, nil
 		// Every start is judged ready afresh (to-be 48 §4).
-		k.Passed, k.Failing, k.Why, k.due = false, 0, "", time.Time{}
+		k.Passed, k.Failing, k.Why, k.Waits, k.due = false, 0, "", nil, time.Time{}
 	}
 
 	// **Held is neither alive nor dead**, and the window ending is a start: judged from a fresh grace.
@@ -562,3 +571,11 @@ func firstLine(s string) string {
 	line, _, _ := strings.Cut(strings.TrimSpace(s), "\n")
 	return line
 }
+
+// waitsOf is what a resource stated Waiting waits for, and nothing for any other state.
+func waitsOf(k *kept) []health.Wait {
+	if k.State != Waiting {
+		return nil
+	}
+	return append([]health.Wait(nil), k.Waits...)
+}

internal/liveness/readiness.go

@@ -14,6 +14,7 @@ import (
 	"strings"
 	"time"
 
+	"git.novox.be/novox/mesh-sdk/go/health"
 	"github.com/novox/mesh-host/internal/declaration"
 )
 
@@ -77,6 +78,10 @@ func ready(k *kept, o Observed, inGrace bool) (string, string) {
 		}
 	}
 	switch {
+	case len(k.Waits) > 0:
+		// **A wait for the operator is a statement, not a flake** (novox/hq ADR 0283): one look that says it
+		// is enough, inside the grace too; the next look that says anything else replaces it.
+		return Waiting, "waits for the operator: " + orSaid(k.Why, "it says what it waits for")
 	case !inGrace && k.Failing >= c.Looks:
 		return Unhealthy, c.Words() + ": " + orSaid(k.Why, "it failed")
 	case k.Passed:
@@ -156,8 +161,8 @@ func (j *Judge) Probe(ctx context.Context) {
 			}
 			go func(d dueLook) {
 				defer func() { <-slots }()
-				ok, why := j.probes().Look(ctx, d.module, d.check)
+				ok, why, waits := j.probes().Look(ctx, d.module, d.check)
-				j.Record(d.id, d.started, ok, why)
+				j.Record(d.id, d.started, ok, why, waits...)
 			}(d)
 		}
 	}
@@ -190,9 +195,10 @@ func (j *Judge) due() []dueLook {
 }
 
 // Record keeps what one of the engine's own looks found, for the next look to fold in: a pass is a pass
-// whenever it came; a failure counts only after the grace. A look begun before the resource's current
+// whenever it came; a failure counts only after the grace; a look that says the resource waits for the
-// start says nothing about it.
+// operator (waits, ADR 0283) is kept as that, in the grace too, and counts as no failing look. A look begun
-func (j *Judge) Record(id string, started time.Time, ok bool, why string) {
+// before the resource's current start says nothing about it.
+func (j *Judge) Record(id string, started time.Time, ok bool, why string, waits ...health.Wait) {
 	j.mu.Lock()
 	defer j.mu.Unlock()
 	k := j.f.Kept[id]
@@ -203,9 +209,12 @@ func (j *Judge) Record(id string, started time.Time, ok bool, why string) {
 	if !k.Started.Equal(started) || k.Check == nil {
 		return
 	}
+	k.Waits = nil
 	switch {
 	case ok:
 		k.Passed, k.Failing, k.Why = true, 0, ""
+	case len(waits) > 0:
+		k.Failing, k.Why, k.Waits = 0, why, append([]health.Wait(nil), waits...)
 	case j.Now().Before(k.Started.Add(j.graceOf(k.Resource))):
 		k.Why = why
 	default:
@@ -227,11 +236,12 @@ type Probes struct {
 	// Host is where an endpoint's port is dialled: this machine, by default its loopback.
 	Host string
 	// AskTool asks a module's tool on this machine's own node tools; nil says it cannot be asked.
-	AskTool func(ctx context.Context, module, tool string) (healthy bool, why string, err error)
+	AskTool func(ctx context.Context, module, tool string) (health.Answer, error)
 }
 
-// Look makes one look of a check, and says whether it passed and, when not, what it found.
+// Look makes one look of a check, and says whether it passed and, when not, what it found — and, for a
-func (p *Probes) Look(ctx context.Context, module string, c *declaration.Health) (bool, string) {
+// tool that answers it waits for the operator (ADR 0283), what it waits for. Only a tool check can wait.
+func (p *Probes) Look(ctx context.Context, module string, c *declaration.Health) (bool, string, []health.Wait) {
 	timeout := c.Within()
 	if timeout <= 0 {
 		timeout = 5 * time.Second
@@ -240,30 +250,33 @@ func (p *Probes) Look(ctx context.Context, module string, c *declaration.Health)
 	defer cancel()
 	switch c.Kind {
 	case declaration.HealthHTTP:
-		return p.http(ctx, c, timeout)
+		ok, why := p.http(ctx, c, timeout)
+		return ok, why, nil
 	case declaration.HealthTCP:
 		conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", p.address(c.Port))
 		if err != nil {
-			return false, dialWords(err, timeout)
+			return false, dialWords(err, timeout), nil
 		}
 		_ = conn.Close()
-		return true, ""
+		return true, "", nil
 	case declaration.HealthTool:
 		if p.AskTool == nil {
-			return false, "its tool cannot be asked: this engine has no link to the node tools"
+			return false, "its tool cannot be asked: this engine has no link to the node tools", nil
 		}
-		healthy, why, err := p.AskTool(ctx, module, c.Tool)
+		a, err := p.AskTool(ctx, module, c.Tool)
 		switch {
 		case errors.Is(err, context.DeadlineExceeded):
-			return false, fmt.Sprintf("no answer within %s", timeout)
+			return false, fmt.Sprintf("no answer within %s", timeout), nil
 		case err != nil:
-			return false, firstLine(err.Error())
+			return false, firstLine(err.Error()), nil
-		case !healthy:
+		case a.State == health.StateWaits && len(a.Waits) > 0:
-			return false, orSaid(why, "it answered not healthy, and not why")
+			return false, orSaid(a.Why, "it waits for the operator, and not why"), a.Waits
+		case !a.Healthy:
+			return false, orSaid(a.Why, "it answered not healthy, and not why"), nil
 		}
-		return true, ""
+		return true, "", nil
 	}
-	return false, "the engine does not make a " + c.Kind + " look itself"
+	return false, "the engine does not make a " + c.Kind + " look itself", nil
 }
 
 func (p *Probes) address(port int) string {

internal/liveness/readiness_test.go

@@ -13,6 +13,7 @@ import (
 	"testing"
 	"time"
 
+	"git.novox.be/novox/mesh-sdk/go/health"
 	"github.com/novox/mesh-host/internal/declaration"
 )
 
@@ -40,8 +41,8 @@ func portOf(t *testing.T, url string) int {
 func lookOnce(t *testing.T, j *Judge) {
 	t.Helper()
 	for _, d := range j.due() {
-		ok, why := j.probes().Look(t.Context(), d.module, d.check)
+		ok, why, waits := j.probes().Look(t.Context(), d.module, d.check)
-		j.Record(d.id, d.started, ok, why)
+		j.Record(d.id, d.started, ok, why, waits...)
 	}
 }
 
@@ -182,11 +183,11 @@ func TestNeverMoreLooksThanTheBudget(t *testing.T) {
 
 func TestAToolCheckIsAskedOfTheNodeTools(t *testing.T) {
 	asked := ""
-	p := &Probes{AskTool: func(_ context.Context, module, tool string) (bool, string, error) {
+	p := &Probes{AskTool: func(_ context.Context, module, tool string) (health.Answer, error) {
 		asked = module + "." + tool
-		return false, "the admin refused the minted secret", nil
+		return health.Unhealthy("the admin refused the minted secret"), nil
 	}}
-	ok, why := p.Look(t.Context(), "keycloak", &declaration.Health{Kind: declaration.HealthTool, Tool: "keycloak_admin_health",
+	ok, why, _ := p.Look(t.Context(), "keycloak", &declaration.Health{Kind: declaration.HealthTool, Tool: "keycloak_admin_health",
 		Interval: "30s", Timeout: "5s", Looks: 2, Grace: "0s"})
 	if ok || asked != "keycloak.keycloak_admin_health" || !strings.Contains(why, "refused the minted secret") {
 		t.Fatalf("asked %q: %v %q", asked, ok, why)
@@ -198,16 +199,16 @@ func TestAToolCheckIsAskedOfTheNodeTools(t *testing.T) {
 func TestAToolCheckThatGetsNoAnswerSaysHowLongItWaited(t *testing.T) {
 	check := &declaration.Health{Kind: declaration.HealthTool, Tool: "sensors_health",
 		Interval: "30s", Timeout: "5s", Looks: 2, Grace: "0s"}
-	p := &Probes{AskTool: func(context.Context, string, string) (bool, string, error) {
+	p := &Probes{AskTool: func(context.Context, string, string) (health.Answer, error) {
-		return false, "", fmt.Errorf("no answer from mesh.mod.sensors.tool.sensors_health.anchor: %w", context.DeadlineExceeded)
+		return health.Answer{}, fmt.Errorf("no answer from mesh.mod.sensors.tool.sensors_health.anchor: %w", context.DeadlineExceeded)
 	}}
-	if ok, why := p.Look(t.Context(), "sensors", check); ok || why != "no answer within 5s" {
+	if ok, why, _ := p.Look(t.Context(), "sensors", check); ok || why != "no answer within 5s" {
 		t.Fatalf("%v %q", ok, why)
 	}
-	p.AskTool = func(context.Context, string, string) (bool, string, error) {
+	p.AskTool = func(context.Context, string, string) (health.Answer, error) {
-		return false, "", errors.New("no link to the bus is open")
+		return health.Answer{}, errors.New("no link to the bus is open")
 	}
-	if ok, why := p.Look(t.Context(), "sensors", check); ok || why != "no link to the bus is open" {
+	if ok, why, _ := p.Look(t.Context(), "sensors", check); ok || why != "no link to the bus is open" {
 		t.Fatalf("%v %q", ok, why)
 	}
 }

internal/liveness/replay_test.go

@@ -168,8 +168,8 @@ func TestReplaySilentWebAppIsSaidUnhealthy(t *testing.T) {
 	looks := 0
 	for time.Since(began) < 2*time.Minute {
 		for _, d := range j.due() {
-			ok, why := j.probes().Look(t.Context(), d.module, d.check)
+			ok, why, waits := j.probes().Look(t.Context(), d.module, d.check)
-			j.Record(d.id, d.started, ok, why)
+			j.Record(d.id, d.started, ok, why, waits...)
 			looks++
 		}
 		st, _ = j.Look(t.Context())
@@ -183,7 +183,7 @@ func TestReplaySilentWebAppIsSaidUnhealthy(t *testing.T) {
 		t.Fatalf("a web application answering nothing was not said unhealthy within two looks (%d looks): %+v", looks, s)
 	}
 	// And a TCP check would have said it reachable: the port is open.
-	if ok, why := tcp.Look(t.Context(), "app", timing(&declaration.Health{Kind: declaration.HealthTCP, Port: published})); !ok {
+	if ok, why, _ := tcp.Look(t.Context(), "app", timing(&declaration.Health{Kind: declaration.HealthTCP, Port: published})); !ok {
 		t.Fatalf("a tcp check could not connect to the silent application: %s", why)
 	}
 	t.Logf("said %s (%s) after %d looks, %s after the judging began; a TCP check connects", s.State, s.Reason, looks,

internal/liveness/waiting_test.go

@@ -0,0 +1,122 @@
+package liveness
+
+import (
+	"context"
+	"encoding/json"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/health"
+	"github.com/novox/mesh-host/internal/declaration"
+)
+
+// A module's tool check may say it waits for the operator (novox/hq ADR 0283, issue 386): a named secret or
+// setting not given yet, and nothing else wrong. The engine states that resource waiting, with its waits, on
+// one look, inside the grace too; the next look that says anything else replaces it.
+
+var password = health.Wait{Part: "the source games", Secret: "smb-password-games", What: "the password of the source games"}
+
+// toolJudge is a judge of one running process whose tool check answers what *answer holds.
+func toolJudge(t *testing.T, answer *[]byte) (*Judge, *clock) {
+	t.Helper()
+	c := &clock{now: t0}
+	rt := &fakeRuntime{containers: map[string]Observed{"watch": running("c1", 0, t0.Format(time.RFC3339Nano))}}
+	j := aJudge(t, rt, c, filepath.Join(t.TempDir(), FileName))
+	j.Probes = &Probes{AskTool: func(context.Context, string, string) (health.Answer, error) {
+		return health.Read(*answer)
+	}}
+	check := &declaration.Health{Kind: declaration.HealthTool, Tool: "mounts_health", Interval: "30s", Timeout: "5s",
+		Looks: 2, Grace: "1m30s"}
+	j.Set([]Resource{{Module: "mounts", ID: "mounts.watch", Kind: KindContainer, Target: "watch", Check: check}})
+	j.Look(t.Context())
+	return j, c
+}
+
+func answerOf(t *testing.T, a health.Answer) []byte {
+	t.Helper()
+	b, err := json.Marshal(a)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return b
+}
+
+func next(t *testing.T, j *Judge, c *clock, by time.Duration) State {
+	t.Helper()
+	c.now = c.now.Add(by)
+	lookOnce(t, j)
+	st, _ := j.Look(t.Context())
+	return stateOf(t, st, "mounts.watch")
+}
+
+func TestAToolThatWaitsIsStatedWaitingWithItsWaitsInsideTheGrace(t *testing.T) {
+	answer := answerOf(t, health.Waiting("the source games waits for its password", password))
+	j, c := toolJudge(t, &answer)
+	s := next(t, j, c, 0)
+	if s.State != Waiting || len(s.Waits) != 1 || s.Waits[0].Secret != "smb-password-games" ||
+		!strings.Contains(s.Reason, "waits for the operator: the source games waits for its password") {
+		t.Fatalf("in its grace, a tool that waits: %+v", s)
+	}
+	if s.Streak != 0 {
+		t.Fatalf("a wait counted as a failing streak: %+v", s)
+	}
+}
+
+func TestAToolThatWaitsStaysWaitingAfterTheGraceAndIsNoFailingLook(t *testing.T) {
+	answer := answerOf(t, health.Waiting("waits", password))
+	j, c := toolJudge(t, &answer)
+	for i := 0; i < 5; i++ {
+		if s := next(t, j, c, 30*time.Second); s.State != Waiting {
+			t.Fatalf("look %d after the grace: %+v", i, s)
+		}
+	}
+}
+
+func TestTheNextHealthyAnswerAfterAWaitIsHealthy(t *testing.T) {
+	answer := answerOf(t, health.Waiting("waits", password))
+	j, c := toolJudge(t, &answer)
+	next(t, j, c, 2*time.Minute)
+	answer = answerOf(t, health.Healthy("every source armed"))
+	if s := next(t, j, c, 30*time.Second); s.State != Healthy || len(s.Waits) != 0 {
+		t.Fatalf("healthy after a wait: %+v", s)
+	}
+}
+
+func TestAnUnhealthyAnswerAfterAWaitFollowsTheFailingLooks(t *testing.T) {
+	answer := answerOf(t, health.Waiting("waits", password))
+	j, c := toolJudge(t, &answer)
+	next(t, j, c, 2*time.Minute)
+	answer = answerOf(t, health.Unhealthy("the share is unreachable"))
+	s := next(t, j, c, 30*time.Second)
+	if s.State == Unhealthy || s.State == Waiting || len(s.Waits) != 0 {
+		t.Fatalf("one failing look after a wait made it %+v", s)
+	}
+	if s = next(t, j, c, 30*time.Second); s.State != Unhealthy || !strings.Contains(s.Reason, "the share is unreachable") {
+		t.Fatalf("two failing looks after a wait: %+v", s)
+	}
+}
+
+func TestAnAnswerWithoutStateIsJudgedAsBefore(t *testing.T) {
+	answer := []byte(`{"healthy":false,"why":"down"}`)
+	j, c := toolJudge(t, &answer)
+	next(t, j, c, 2*time.Minute)
+	if s := next(t, j, c, 30*time.Second); s.State != Unhealthy || len(s.Waits) != 0 {
+		t.Fatalf("an old unhealthy answer: %+v", s)
+	}
+	answer = []byte(`{"healthy":true,"why":"ok"}`)
+	if s := next(t, j, c, 30*time.Second); s.State != Healthy {
+		t.Fatalf("an old healthy answer: %+v", s)
+	}
+}
+
+func TestAMalformedWaitIsUnhealthyAndExcusesNothing(t *testing.T) {
+	answer := []byte(`{"state":"waits","healthy":false,"why":"it waits","waits":[{"part":"p","what":"the password"}]}`)
+	j, c := toolJudge(t, &answer)
+	next(t, j, c, 2*time.Minute)
+	s := next(t, j, c, 30*time.Second)
+	if s.State != Unhealthy || len(s.Waits) != 0 || !strings.Contains(s.Reason, "neither a secret nor a setting") {
+		t.Fatalf("a wait naming nothing: %+v", s)
+	}
+}