State a tool check that waits for the operator as waiting (issue 386, ADR 0283)

A module whose only fault was a secret the operator had not given yet was
stated unhealthy, failed its first-node gate and was put back. The engine
now reads the tool's answer through mesh-sdk go/health and states such a
resource waiting, with what it waits for, on one look; a malformed wait
stays unhealthy.

internal/liveness/readiness.go

@@ -14,6 +14,7 @@ import (
 	"strings"
 	"time"
 
+	"git.novox.be/novox/mesh-sdk/go/health"
 	"github.com/novox/mesh-host/internal/declaration"
 )
 
@@ -77,6 +78,10 @@ func ready(k *kept, o Observed, inGrace bool) (string, string) {
 		}
 	}
 	switch {
+	case len(k.Waits) > 0:
+		// **A wait for the operator is a statement, not a flake** (novox/hq ADR 0283): one look that says it
+		// is enough, inside the grace too; the next look that says anything else replaces it.
+		return Waiting, "waits for the operator: " + orSaid(k.Why, "it says what it waits for")
 	case !inGrace && k.Failing >= c.Looks:
 		return Unhealthy, c.Words() + ": " + orSaid(k.Why, "it failed")
 	case k.Passed:
@@ -156,8 +161,8 @@ func (j *Judge) Probe(ctx context.Context) {
 			}
 			go func(d dueLook) {
 				defer func() { <-slots }()
-				ok, why := j.probes().Look(ctx, d.module, d.check)
-				j.Record(d.id, d.started, ok, why)
+				ok, why, waits := j.probes().Look(ctx, d.module, d.check)
+				j.Record(d.id, d.started, ok, why, waits...)
 			}(d)
 		}
 	}
@@ -190,9 +195,10 @@ func (j *Judge) due() []dueLook {
 }
 
 // Record keeps what one of the engine's own looks found, for the next look to fold in: a pass is a pass
-// whenever it came; a failure counts only after the grace. A look begun before the resource's current
-// start says nothing about it.
-func (j *Judge) Record(id string, started time.Time, ok bool, why string) {
+// whenever it came; a failure counts only after the grace; a look that says the resource waits for the
+// operator (waits, ADR 0283) is kept as that, in the grace too, and counts as no failing look. A look begun
+// before the resource's current start says nothing about it.
+func (j *Judge) Record(id string, started time.Time, ok bool, why string, waits ...health.Wait) {
 	j.mu.Lock()
 	defer j.mu.Unlock()
 	k := j.f.Kept[id]
@@ -203,9 +209,12 @@ func (j *Judge) Record(id string, started time.Time, ok bool, why string) {
 	if !k.Started.Equal(started) || k.Check == nil {
 		return
 	}
+	k.Waits = nil
 	switch {
 	case ok:
 		k.Passed, k.Failing, k.Why = true, 0, ""
+	case len(waits) > 0:
+		k.Failing, k.Why, k.Waits = 0, why, append([]health.Wait(nil), waits...)
 	case j.Now().Before(k.Started.Add(j.graceOf(k.Resource))):
 		k.Why = why
 	default:
@@ -227,11 +236,12 @@ type Probes struct {
 	// Host is where an endpoint's port is dialled: this machine, by default its loopback.
 	Host string
 	// AskTool asks a module's tool on this machine's own node tools; nil says it cannot be asked.
-	AskTool func(ctx context.Context, module, tool string) (healthy bool, why string, err error)
+	AskTool func(ctx context.Context, module, tool string) (health.Answer, error)
 }
 
-// Look makes one look of a check, and says whether it passed and, when not, what it found.
-func (p *Probes) Look(ctx context.Context, module string, c *declaration.Health) (bool, string) {
+// Look makes one look of a check, and says whether it passed and, when not, what it found — and, for a
+// tool that answers it waits for the operator (ADR 0283), what it waits for. Only a tool check can wait.
+func (p *Probes) Look(ctx context.Context, module string, c *declaration.Health) (bool, string, []health.Wait) {
 	timeout := c.Within()
 	if timeout <= 0 {
 		timeout = 5 * time.Second
@@ -240,30 +250,33 @@ func (p *Probes) Look(ctx context.Context, module string, c *declaration.Health)
 	defer cancel()
 	switch c.Kind {
 	case declaration.HealthHTTP:
-		return p.http(ctx, c, timeout)
+		ok, why := p.http(ctx, c, timeout)
+		return ok, why, nil
 	case declaration.HealthTCP:
 		conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", p.address(c.Port))
 		if err != nil {
-			return false, dialWords(err, timeout)
+			return false, dialWords(err, timeout), nil
 		}
 		_ = conn.Close()
-		return true, ""
+		return true, "", nil
 	case declaration.HealthTool:
 		if p.AskTool == nil {
-			return false, "its tool cannot be asked: this engine has no link to the node tools"
+			return false, "its tool cannot be asked: this engine has no link to the node tools", nil
 		}
-		healthy, why, err := p.AskTool(ctx, module, c.Tool)
+		a, err := p.AskTool(ctx, module, c.Tool)
 		switch {
 		case errors.Is(err, context.DeadlineExceeded):
-			return false, fmt.Sprintf("no answer within %s", timeout)
+			return false, fmt.Sprintf("no answer within %s", timeout), nil
 		case err != nil:
-			return false, firstLine(err.Error())
-		case !healthy:
-			return false, orSaid(why, "it answered not healthy, and not why")
+			return false, firstLine(err.Error()), nil
+		case a.State == health.StateWaits && len(a.Waits) > 0:
+			return false, orSaid(a.Why, "it waits for the operator, and not why"), a.Waits
+		case !a.Healthy:
+			return false, orSaid(a.Why, "it answered not healthy, and not why"), nil
 		}
-		return true, ""
+		return true, "", nil
 	}
-	return false, "the engine does not make a " + c.Kind + " look itself"
+	return false, "the engine does not make a " + c.Kind + " look itself", nil
 }
 
 func (p *Probes) address(port int) string {