State a tool check that waits for the operator as waiting (issue 386, ADR 0283)

A module whose only fault was a secret the operator had not given yet was
stated unhealthy, failed its first-node gate and was put back. The engine
now reads the tool's answer through mesh-sdk go/health and states such a
resource waiting, with what it waits for, on one look; a malformed wait
stays unhealthy.

internal/link/asktool_nats_test.go

@@ -29,11 +29,11 @@ func TestNatsAToolCheckIsAskedOnTheLink(t *testing.T) {
 	q.attach(t.Context(), &natsLink{conn: conn, js: js, node: node})
 	ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second)
 	defer cancel()
-	healthy, why, err := q.AskTool(ctx, "sensors", "sensors_health")
-	if err != nil || !healthy {
-		t.Fatalf("a served healthy tool read as %v %q (%v)", healthy, why, err)
+	a, err := q.AskTool(ctx, "sensors", "sensors_health")
+	if err != nil || !a.Healthy {
+		t.Fatalf("a served healthy tool read as %+v (%v)", a, err)
 	}
-	_, _, err = q.AskTool(ctx, "sensors", "nothing_serves_this")
+	_, err = q.AskTool(ctx, "sensors", "nothing_serves_this")
 	if err == nil || !strings.Contains(err.Error(), "nothing on this machine answers") {
 		t.Fatalf("a tool nothing serves read as %v", err)
 	}

internal/link/health_test.go

@@ -62,8 +62,8 @@ func TestAToolsAnswerIsHealthyOnlyWhenItSaysSo(t *testing.T) {
 		`{"error":"503 no responders"}`:                                     false,
 		`{"result":"yes"}`:                                                  false,
 	} {
-		got, _, err := ReadToolAnswer([]byte(reply))
-		if err != nil || got != want {
+		got, err := ReadToolAnswer([]byte(reply))
+		if err != nil || got.Healthy != want {
 			t.Errorf("%s read as %v (%v)", reply, got, err)
 		}
 	}
@@ -78,13 +78,13 @@ func TestAToolsAnswerIsHealthyOnlyWhenItSaysSo(t *testing.T) {
 // ask, the check fails and says which.
 func TestAToolCheckIsAskedOnTheLinkTheEngineHolds(t *testing.T) {
 	q := &Queue{Membership: Membership{Node: "anchor"}}
-	if _, _, err := q.AskTool(t.Context(), "sensors", "sensors_health"); err == nil || err.Error() != "no link to the bus is open" {
+	if _, err := q.AskTool(t.Context(), "sensors", "sensors_health"); err == nil || err.Error() != "no link to the bus is open" {
 		t.Fatalf("with no link open: %v", err)
 	}
 
 	quiet := newQuietLink()
 	q.attach(t.Context(), quiet)
-	if _, _, err := q.AskTool(t.Context(), "sensors", "sensors_health"); err == nil || err.Error() != "the link to the bus open now cannot ask a tool" {
+	if _, err := q.AskTool(t.Context(), "sensors", "sensors_health"); err == nil || err.Error() != "the link to the bus open now cannot ask a tool" {
 		t.Fatalf("on a link that cannot ask: %v", err)
 	}
 	q.detach(quiet)
@@ -93,7 +93,7 @@ func TestAToolCheckIsAskedOnTheLinkTheEngineHolds(t *testing.T) {
 	// there), never stop at the queue's assertion.
 	held := &natsLink{node: "anchor"}
 	q.attach(t.Context(), held)
-	_, _, err := q.AskTool(t.Context(), "sensors", "sensors_health")
+	_, err := q.AskTool(t.Context(), "sensors", "sensors_health")
 	if err == nil {
 		t.Fatal("asked on no connection and was answered")
 	}
@@ -101,3 +101,18 @@ func TestAToolCheckIsAskedOnTheLinkTheEngineHolds(t *testing.T) {
 		t.Fatalf("the NATS link was not asked: %v", err)
 	}
 }
+
+// A tool may answer that it waits for the operator (novox/hq ADR 0283): read with what it waits for, and not
+// healthy, so nothing reading `healthy` alone takes it for a pass.
+func TestAToolsWaitIsReadWithWhatItWaitsFor(t *testing.T) {
+	got, err := ReadToolAnswer([]byte(`{"result":{"state":"waits","healthy":false,"why":"the source games waits",` +
+		`"waits":[{"part":"the source games","secret":"smb-password-games","what":"the password of the source games"}]}}`))
+	if err != nil || got.Healthy || got.State != "waits" || len(got.Waits) != 1 || got.Waits[0].Secret != "smb-password-games" {
+		t.Fatalf("read as %+v (%v)", got, err)
+	}
+	h := ResourceHealth{Module: "mounts", Resource: "mounts.watch", State: StateWaiting, Waits: got.Waits}
+	b, _ := json.Marshal(h)
+	if !strings.Contains(string(b), `"state":"waiting"`) || !strings.Contains(string(b), `"waits":[{"part":"the source games","secret":"smb-password-games"`) {
+		t.Fatalf("said on the wire as %s", b)
+	}
+}

internal/link/messages.go

@@ -4,6 +4,8 @@ import (
 	"strconv"
 	"strings"
 	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/health"
 )
 
 // The wire formats shared with the control plane, which defines them separately because this
@@ -335,6 +337,10 @@ const (
 	StateHeld = "held"
 	// StateUnknown is a resource whose state could not be read.
 	StateUnknown = "unknown"
+	// StateWaiting is a resource whose tool check says it waits for the operator — a named own secret or
+	// setting of the module not given yet — and nothing else is wrong (novox/hq ADR 0283). Not a fault: the
+	// controller checks the wait, and an older controller reads it as not healthy and raises nothing.
+	StateWaiting = "waiting"
 )
 
 // The reasons an unhealthy resource is said with, when it is liveness that judged it.
@@ -376,6 +382,9 @@ type ResourceHealth struct {
 	// Search is how far the setuid search got, on an account's verdict that waits for it (novox/hq issue
 	// 361): Reason says the same in words. Absent otherwise.
 	Search *SetuidSearch `json:"search,omitempty"`
+	// Waits is what a resource stated waiting waits for (novox/hq ADR 0283): each one own secret or one
+	// setting of the module by name, the part, and what the operator gives, in words. Absent otherwise.
+	Waits []health.Wait `json:"waits,omitempty"`
 }
 
 // SetuidSearch is how far a running search for setuid programs got, as of its last checkpoint. Counts, times

internal/link/queue.go

@@ -9,6 +9,8 @@ import (
 	"fmt"
 	"sync"
 	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/health"
 )
 
 // Queue is the node-engine's one apply queue (novox/hq to-be 45 §6, ADR 0227 rule 1).
@@ -531,21 +533,21 @@ func (q *Queue) timeout() time.Duration {
 // AskTool asks a module's tool on this machine's own node tools, as a declared health check (novox/hq ADR
 // 0240, to-be 48 §3), and reads its answer: healthy or not, with why. An error when there is no link open
 // or nothing answered — said as the check's failure, never as healthy.
-func (q *Queue) AskTool(ctx context.Context, module, tool string) (bool, string, error) {
+func (q *Queue) AskTool(ctx context.Context, module, tool string) (health.Answer, error) {
 	q.init()
 	q.mu.Lock()
 	bus := q.bus
 	q.mu.Unlock()
 	if bus == nil {
-		return false, "", errors.New("no link to the bus is open")
+		return health.Answer{}, errors.New("no link to the bus is open")
 	}
 	asker, ok := bus.(ToolBus)
 	if !ok {
-		return false, "", errors.New("the link to the bus open now cannot ask a tool")
+		return health.Answer{}, errors.New("the link to the bus open now cannot ask a tool")
 	}
 	reply, err := asker.Ask(ctx, ToolSubject(module, tool, q.Membership.Node), []byte("{}"))
 	if err != nil {
-		return false, "", err
+		return health.Answer{}, err
 	}
 	return ReadToolAnswer(reply)
 }
@@ -572,24 +574,23 @@ func (q *Queue) AskController(ctx context.Context, body []byte) ([]byte, error)
 var ErrNotAsked = errors.New("not asked")
 
 // ReadToolAnswer reads a tool's reply envelope — `{result, error}`, as the node tools answer every call —
-// as a health answer: an error is not healthy, and so is a result that does not say `healthy: true`.
-func ReadToolAnswer(reply []byte) (bool, string, error) {
+// as a health answer, through the SDK's reader (novox/hq ADR 0283): healthy, unhealthy, or waits for the
+// operator with what it waits for. An error, and a result that is not an answer, are unhealthy; a wait the
+// SDK cannot check is unhealthy too, so a malformed one never excuses anything.
+func ReadToolAnswer(reply []byte) (health.Answer, error) {
 	var env struct {
 		Result json.RawMessage `json:"result"`
 		Error  string          `json:"error"`
 	}
 	if err := json.Unmarshal(reply, &env); err != nil {
-		return false, "", fmt.Errorf("its answer is not one: %w", err)
+		return health.Answer{}, fmt.Errorf("its answer is not one: %w", err)
 	}
 	if env.Error != "" {
-		return false, env.Error, nil
+		return health.Unhealthy(env.Error), nil
 	}
-	var a struct {
-		Healthy bool   `json:"healthy"`
-		Why     string `json:"why"`
+	a, err := health.Read(env.Result)
+	if err != nil {
+		return health.Unhealthy("it answered something other than healthy or not"), nil
 	}
-	if err := json.Unmarshal(env.Result, &a); err != nil {
-		return false, "it answered something other than healthy or not", nil
-	}
-	return a.Healthy, a.Why, nil
+	return a, nil
 }

internal/link/replay331_test.go

@@ -15,7 +15,7 @@ import (
 func TestReplay331(t *testing.T) {
 	q := &Queue{Membership: Membership{Node: "laptop"}}
 	q.attach(t.Context(), &natsLink{node: "laptop"})
-	_, _, err := q.AskTool(t.Context(), "sensors", "sensors_health")
+	_, err := q.AskTool(t.Context(), "sensors", "sensors_health")
 	if err == nil {
 		t.Fatal("asked on no connection and was answered")
 	}