jochen
3abb07f1e2
A module whose only fault was a secret the operator had not given yet was stated unhealthy, failed its first-node gate and was put back. The engine now reads the tool's answer through mesh-sdk go/health and states such a resource waiting, with what it waits for, on one look; a malformed wait stays unhealthy.
119 lines
5.0 KiB
Go
119 lines
5.0 KiB
Go
package link
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// A health statement is said on the link open now, as the machine's own word on its own subject, and
|
|
// not said — for the caller to say again — while no link is open (novox/hq ADR 0240, to-be 48 §4).
|
|
|
|
// sayingLink is a link that can say health.
|
|
type sayingLink struct {
|
|
*quietLink
|
|
said [][]byte
|
|
}
|
|
|
|
func (l *sayingLink) Health(_ context.Context, node string, body []byte) error {
|
|
l.said = append(l.said, body)
|
|
return nil
|
|
}
|
|
|
|
func TestAHealthStatementIsSaidOnTheLinkOpenNow(t *testing.T) {
|
|
q := &Queue{Membership: Membership{Node: "anchor"}}
|
|
h := Health{Contract: LivenessContract, At: time.Now().UTC(),
|
|
Resources: []ResourceHealth{{Module: "letta", Resource: "letta.server", State: StateUnhealthy, Reason: ReasonRestarting}}}
|
|
if q.SayHealth(t.Context(), h) {
|
|
t.Fatal("said with no link open")
|
|
}
|
|
l := &sayingLink{quietLink: newQuietLink()}
|
|
q.attach(t.Context(), l)
|
|
if !q.SayHealth(t.Context(), h) || len(l.said) != 1 {
|
|
t.Fatalf("not said on the open link: %d", len(l.said))
|
|
}
|
|
var got HealthSaid
|
|
if err := json.Unmarshal(l.said[0], &got); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got.Node != "anchor" || len(got.Health.Resources) != 1 || got.Health.Resources[0].State != StateUnhealthy {
|
|
t.Fatalf("said %+v", got)
|
|
}
|
|
// A link that cannot say it is not an error: the next report carries the statement.
|
|
q.detach(l)
|
|
q.attach(t.Context(), newQuietLink())
|
|
if q.SayHealth(t.Context(), h) {
|
|
t.Fatal("said on a link that cannot")
|
|
}
|
|
if HealthSubject("anchor") != "mesh.control.anchor.health" {
|
|
t.Fatalf("the subject %s is not inside the host's own grant", HealthSubject("anchor"))
|
|
}
|
|
}
|
|
|
|
// A declared tool check reads the node tools' reply envelope: healthy only when the tool says so (novox/hq
|
|
// ADR 0240, to-be 48 §2), and asked on this machine's own instance (to-be 48 §3).
|
|
func TestAToolsAnswerIsHealthyOnlyWhenItSaysSo(t *testing.T) {
|
|
for reply, want := range map[string]bool{
|
|
`{"result":{"healthy":true},"node":"anchor"}`: true,
|
|
`{"result":{"healthy":false,"why":"the admin refused the secret"}}`: false,
|
|
`{"result":{"status":"fine"}}`: false,
|
|
`{"error":"503 no responders"}`: false,
|
|
`{"result":"yes"}`: false,
|
|
} {
|
|
got, err := ReadToolAnswer([]byte(reply))
|
|
if err != nil || got.Healthy != want {
|
|
t.Errorf("%s read as %v (%v)", reply, got, err)
|
|
}
|
|
}
|
|
if ToolSubject("keycloak", "keycloak_admin_health", "anchor") != "mesh.mod.keycloak.tool.keycloak_admin_health.anchor" {
|
|
t.Errorf("asked on %s", ToolSubject("keycloak", "keycloak_admin_health", "anchor"))
|
|
}
|
|
}
|
|
|
|
// A declared tool check is asked on the link the node-engine actually holds — the NATS link — and not
|
|
// refused as if none were open (2026-10-08: every tool check read "no link to the bus is open" while
|
|
// the link was up, because the NATS link could not Ask). With no link open, and with a link that cannot
|
|
// ask, the check fails and says which.
|
|
func TestAToolCheckIsAskedOnTheLinkTheEngineHolds(t *testing.T) {
|
|
q := &Queue{Membership: Membership{Node: "anchor"}}
|
|
if _, err := q.AskTool(t.Context(), "sensors", "sensors_health"); err == nil || err.Error() != "no link to the bus is open" {
|
|
t.Fatalf("with no link open: %v", err)
|
|
}
|
|
|
|
quiet := newQuietLink()
|
|
q.attach(t.Context(), quiet)
|
|
if _, err := q.AskTool(t.Context(), "sensors", "sensors_health"); err == nil || err.Error() != "the link to the bus open now cannot ask a tool" {
|
|
t.Fatalf("on a link that cannot ask: %v", err)
|
|
}
|
|
q.detach(quiet)
|
|
|
|
// The link Open returns, with no connection under it: the question must reach its Ask (and fail
|
|
// there), never stop at the queue's assertion.
|
|
held := &natsLink{node: "anchor"}
|
|
q.attach(t.Context(), held)
|
|
_, err := q.AskTool(t.Context(), "sensors", "sensors_health")
|
|
if err == nil {
|
|
t.Fatal("asked on no connection and was answered")
|
|
}
|
|
if strings.Contains(err.Error(), "no link") || strings.Contains(err.Error(), "cannot ask a tool") {
|
|
t.Fatalf("the NATS link was not asked: %v", err)
|
|
}
|
|
}
|
|
|
|
// A tool may answer that it waits for the operator (novox/hq ADR 0283): read with what it waits for, and not
|
|
// healthy, so nothing reading `healthy` alone takes it for a pass.
|
|
func TestAToolsWaitIsReadWithWhatItWaitsFor(t *testing.T) {
|
|
got, err := ReadToolAnswer([]byte(`{"result":{"state":"waits","healthy":false,"why":"the source games waits",` +
|
|
`"waits":[{"part":"the source games","secret":"smb-password-games","what":"the password of the source games"}]}}`))
|
|
if err != nil || got.Healthy || got.State != "waits" || len(got.Waits) != 1 || got.Waits[0].Secret != "smb-password-games" {
|
|
t.Fatalf("read as %+v (%v)", got, err)
|
|
}
|
|
h := ResourceHealth{Module: "mounts", Resource: "mounts.watch", State: StateWaiting, Waits: got.Waits}
|
|
b, _ := json.Marshal(h)
|
|
if !strings.Contains(string(b), `"state":"waiting"`) || !strings.Contains(string(b), `"waits":[{"part":"the source games","secret":"smb-password-games"`) {
|
|
t.Fatalf("said on the wire as %s", b)
|
|
}
|
|
}
|