From 3abb07f1e25b8da73675b341aa2e57986ce445dd Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 10 Oct 2026 21:11:02 +0200 Subject: [PATCH 1/2] State a tool check that waits for the operator as waiting (issue 386, ADR 0283) A module whose only fault was a secret the operator had not given yet was stated unhealthy, failed its first-node gate and was put back. The engine now reads the tool's answer through mesh-sdk go/health and states such a resource waiting, with what it waits for, on one look; a malformed wait stays unhealthy. --- cmd/mesh-host/main.go | 3 +- go.mod | 2 +- go.sum | 4 + internal/link/asktool_nats_test.go | 8 +- internal/link/health_test.go | 25 ++++-- internal/link/messages.go | 9 ++ internal/link/queue.go | 31 +++---- internal/link/replay331_test.go | 2 +- internal/liveness/liveness.go | 23 +++++- internal/liveness/readiness.go | 51 +++++++----- internal/liveness/readiness_test.go | 23 +++--- internal/liveness/replay_test.go | 6 +- internal/liveness/waiting_test.go | 122 ++++++++++++++++++++++++++++ 13 files changed, 246 insertions(+), 63 deletions(-) create mode 100644 internal/liveness/waiting_test.go diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index 98ff07d..cc11fc1 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -1619,7 +1619,8 @@ func healthAsReported(st liveness.Statement, ns *network.Statement) *link.Health for _, r := range st.Resources { h.Resources = append(h.Resources, link.ResourceHealth{Module: r.Module, Resource: r.ID, Kind: r.Kind, Target: r.Target, State: r.State, Reason: r.Reason, Since: r.Since.UTC(), Streak: r.Streak, - Restarts: r.Restarts, Check: r.CheckOf(), Needs: r.NeedsOf(), Account: userOf(r.Scope, r.User)}) + Restarts: r.Restarts, Check: r.CheckOf(), Needs: r.NeedsOf(), Account: userOf(r.Scope, r.User), + Waits: r.Waits}) } if ns != nil && ns.State != "" { h.Network = &link.NetworkHealth{State: ns.State, Since: ns.Since.UTC(), Parts: []link.NetworkPart{}} diff --git a/go.mod b/go.mod index 8127078..627b0d9 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-host go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.11-0.20261009143344-f047d0a4a970 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c github.com/nats-io/nats.go v1.54.0 golang.org/x/crypto v0.57.0 golang.org/x/sys v0.48.0 diff --git a/go.sum b/go.sum index cee9b08..315ddeb 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,9 @@ git.novox.be/novox/mesh-sdk/go v0.1.11-0.20261009143344-f047d0a4a970 h1:9tFDQsgmI+4X7/BpZGXIr+HemPKE7YddYGqWV0lINAI= git.novox.be/novox/mesh-sdk/go v0.1.11-0.20261009143344-f047d0a4a970/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010190644-622b6b4f556d h1:BR8uwv4BqyLaBwITuLylrd9DRUnu0D5DjJw0RuQSXbk= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010190644-622b6b4f556d/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c h1:l5onJwoIeH8yE/PjVlEeoPyXBsHp2NQiWLllz2fwX7s= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= diff --git a/internal/link/asktool_nats_test.go b/internal/link/asktool_nats_test.go index e9caa58..ca34dbd 100644 --- a/internal/link/asktool_nats_test.go +++ b/internal/link/asktool_nats_test.go @@ -29,11 +29,11 @@ func TestNatsAToolCheckIsAskedOnTheLink(t *testing.T) { q.attach(t.Context(), &natsLink{conn: conn, js: js, node: node}) ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second) defer cancel() - healthy, why, err := q.AskTool(ctx, "sensors", "sensors_health") - if err != nil || !healthy { - t.Fatalf("a served healthy tool read as %v %q (%v)", healthy, why, err) + a, err := q.AskTool(ctx, "sensors", "sensors_health") + if err != nil || !a.Healthy { + t.Fatalf("a served healthy tool read as %+v (%v)", a, err) } - _, _, err = q.AskTool(ctx, "sensors", "nothing_serves_this") + _, err = q.AskTool(ctx, "sensors", "nothing_serves_this") if err == nil || !strings.Contains(err.Error(), "nothing on this machine answers") { t.Fatalf("a tool nothing serves read as %v", err) } diff --git a/internal/link/health_test.go b/internal/link/health_test.go index dd701ae..3132c4b 100644 --- a/internal/link/health_test.go +++ b/internal/link/health_test.go @@ -62,8 +62,8 @@ func TestAToolsAnswerIsHealthyOnlyWhenItSaysSo(t *testing.T) { `{"error":"503 no responders"}`: false, `{"result":"yes"}`: false, } { - got, _, err := ReadToolAnswer([]byte(reply)) - if err != nil || got != want { + got, err := ReadToolAnswer([]byte(reply)) + if err != nil || got.Healthy != want { t.Errorf("%s read as %v (%v)", reply, got, err) } } @@ -78,13 +78,13 @@ func TestAToolsAnswerIsHealthyOnlyWhenItSaysSo(t *testing.T) { // ask, the check fails and says which. func TestAToolCheckIsAskedOnTheLinkTheEngineHolds(t *testing.T) { q := &Queue{Membership: Membership{Node: "anchor"}} - if _, _, err := q.AskTool(t.Context(), "sensors", "sensors_health"); err == nil || err.Error() != "no link to the bus is open" { + if _, err := q.AskTool(t.Context(), "sensors", "sensors_health"); err == nil || err.Error() != "no link to the bus is open" { t.Fatalf("with no link open: %v", err) } quiet := newQuietLink() q.attach(t.Context(), quiet) - if _, _, err := q.AskTool(t.Context(), "sensors", "sensors_health"); err == nil || err.Error() != "the link to the bus open now cannot ask a tool" { + if _, err := q.AskTool(t.Context(), "sensors", "sensors_health"); err == nil || err.Error() != "the link to the bus open now cannot ask a tool" { t.Fatalf("on a link that cannot ask: %v", err) } q.detach(quiet) @@ -93,7 +93,7 @@ func TestAToolCheckIsAskedOnTheLinkTheEngineHolds(t *testing.T) { // there), never stop at the queue's assertion. held := &natsLink{node: "anchor"} q.attach(t.Context(), held) - _, _, err := q.AskTool(t.Context(), "sensors", "sensors_health") + _, err := q.AskTool(t.Context(), "sensors", "sensors_health") if err == nil { t.Fatal("asked on no connection and was answered") } @@ -101,3 +101,18 @@ func TestAToolCheckIsAskedOnTheLinkTheEngineHolds(t *testing.T) { t.Fatalf("the NATS link was not asked: %v", err) } } + +// A tool may answer that it waits for the operator (novox/hq ADR 0283): read with what it waits for, and not +// healthy, so nothing reading `healthy` alone takes it for a pass. +func TestAToolsWaitIsReadWithWhatItWaitsFor(t *testing.T) { + got, err := ReadToolAnswer([]byte(`{"result":{"state":"waits","healthy":false,"why":"the source games waits",` + + `"waits":[{"part":"the source games","secret":"smb-password-games","what":"the password of the source games"}]}}`)) + if err != nil || got.Healthy || got.State != "waits" || len(got.Waits) != 1 || got.Waits[0].Secret != "smb-password-games" { + t.Fatalf("read as %+v (%v)", got, err) + } + h := ResourceHealth{Module: "mounts", Resource: "mounts.watch", State: StateWaiting, Waits: got.Waits} + b, _ := json.Marshal(h) + if !strings.Contains(string(b), `"state":"waiting"`) || !strings.Contains(string(b), `"waits":[{"part":"the source games","secret":"smb-password-games"`) { + t.Fatalf("said on the wire as %s", b) + } +} diff --git a/internal/link/messages.go b/internal/link/messages.go index b36a7e1..68ff0bc 100644 --- a/internal/link/messages.go +++ b/internal/link/messages.go @@ -4,6 +4,8 @@ import ( "strconv" "strings" "time" + + "git.novox.be/novox/mesh-sdk/go/health" ) // The wire formats shared with the control plane, which defines them separately because this @@ -335,6 +337,10 @@ const ( StateHeld = "held" // StateUnknown is a resource whose state could not be read. StateUnknown = "unknown" + // StateWaiting is a resource whose tool check says it waits for the operator — a named own secret or + // setting of the module not given yet — and nothing else is wrong (novox/hq ADR 0283). Not a fault: the + // controller checks the wait, and an older controller reads it as not healthy and raises nothing. + StateWaiting = "waiting" ) // The reasons an unhealthy resource is said with, when it is liveness that judged it. @@ -376,6 +382,9 @@ type ResourceHealth struct { // Search is how far the setuid search got, on an account's verdict that waits for it (novox/hq issue // 361): Reason says the same in words. Absent otherwise. Search *SetuidSearch `json:"search,omitempty"` + // Waits is what a resource stated waiting waits for (novox/hq ADR 0283): each one own secret or one + // setting of the module by name, the part, and what the operator gives, in words. Absent otherwise. + Waits []health.Wait `json:"waits,omitempty"` } // SetuidSearch is how far a running search for setuid programs got, as of its last checkpoint. Counts, times diff --git a/internal/link/queue.go b/internal/link/queue.go index 38a777d..539189c 100644 --- a/internal/link/queue.go +++ b/internal/link/queue.go @@ -9,6 +9,8 @@ import ( "fmt" "sync" "time" + + "git.novox.be/novox/mesh-sdk/go/health" ) // Queue is the node-engine's one apply queue (novox/hq to-be 45 §6, ADR 0227 rule 1). @@ -531,21 +533,21 @@ func (q *Queue) timeout() time.Duration { // AskTool asks a module's tool on this machine's own node tools, as a declared health check (novox/hq ADR // 0240, to-be 48 §3), and reads its answer: healthy or not, with why. An error when there is no link open // or nothing answered — said as the check's failure, never as healthy. -func (q *Queue) AskTool(ctx context.Context, module, tool string) (bool, string, error) { +func (q *Queue) AskTool(ctx context.Context, module, tool string) (health.Answer, error) { q.init() q.mu.Lock() bus := q.bus q.mu.Unlock() if bus == nil { - return false, "", errors.New("no link to the bus is open") + return health.Answer{}, errors.New("no link to the bus is open") } asker, ok := bus.(ToolBus) if !ok { - return false, "", errors.New("the link to the bus open now cannot ask a tool") + return health.Answer{}, errors.New("the link to the bus open now cannot ask a tool") } reply, err := asker.Ask(ctx, ToolSubject(module, tool, q.Membership.Node), []byte("{}")) if err != nil { - return false, "", err + return health.Answer{}, err } return ReadToolAnswer(reply) } @@ -572,24 +574,23 @@ func (q *Queue) AskController(ctx context.Context, body []byte) ([]byte, error) var ErrNotAsked = errors.New("not asked") // ReadToolAnswer reads a tool's reply envelope — `{result, error}`, as the node tools answer every call — -// as a health answer: an error is not healthy, and so is a result that does not say `healthy: true`. -func ReadToolAnswer(reply []byte) (bool, string, error) { +// as a health answer, through the SDK's reader (novox/hq ADR 0283): healthy, unhealthy, or waits for the +// operator with what it waits for. An error, and a result that is not an answer, are unhealthy; a wait the +// SDK cannot check is unhealthy too, so a malformed one never excuses anything. +func ReadToolAnswer(reply []byte) (health.Answer, error) { var env struct { Result json.RawMessage `json:"result"` Error string `json:"error"` } if err := json.Unmarshal(reply, &env); err != nil { - return false, "", fmt.Errorf("its answer is not one: %w", err) + return health.Answer{}, fmt.Errorf("its answer is not one: %w", err) } if env.Error != "" { - return false, env.Error, nil + return health.Unhealthy(env.Error), nil } - var a struct { - Healthy bool `json:"healthy"` - Why string `json:"why"` + a, err := health.Read(env.Result) + if err != nil { + return health.Unhealthy("it answered something other than healthy or not"), nil } - if err := json.Unmarshal(env.Result, &a); err != nil { - return false, "it answered something other than healthy or not", nil - } - return a.Healthy, a.Why, nil + return a, nil } diff --git a/internal/link/replay331_test.go b/internal/link/replay331_test.go index 3c5b677..468af9f 100644 --- a/internal/link/replay331_test.go +++ b/internal/link/replay331_test.go @@ -15,7 +15,7 @@ import ( func TestReplay331(t *testing.T) { q := &Queue{Membership: Membership{Node: "laptop"}} q.attach(t.Context(), &natsLink{node: "laptop"}) - _, _, err := q.AskTool(t.Context(), "sensors", "sensors_health") + _, err := q.AskTool(t.Context(), "sensors", "sensors_health") if err == nil { t.Fatal("asked on no connection and was answered") } diff --git a/internal/liveness/liveness.go b/internal/liveness/liveness.go index 506ccad..140d64a 100644 --- a/internal/liveness/liveness.go +++ b/internal/liveness/liveness.go @@ -39,6 +39,7 @@ import ( "sync" "time" + "git.novox.be/novox/mesh-sdk/go/health" "github.com/novox/mesh-host/internal/declaration" ) @@ -66,6 +67,9 @@ const ( Starting = "starting" Held = "held" Unknown = "unknown" + // Waiting is a resource whose tool check says it waits for the operator, and nothing else is wrong + // (novox/hq ADR 0283): a statement, not a fault, said with what it waits for. + Waiting = "waiting" ReasonDown = "down" ReasonRestarting = "restarting" @@ -175,6 +179,8 @@ type State struct { Since time.Time `json:"since"` Streak int `json:"streak,omitempty"` Restarts int `json:"restarts,omitempty"` + // Waits is what a resource stated Waiting waits for (ADR 0283). + Waits []health.Wait `json:"waits,omitempty"` } // CheckOf and NeedsOf are a stated resource's declared check, in a word, and the provision it exercises. @@ -235,6 +241,9 @@ type kept struct { Passed bool `json:"passed,omitempty"` Failing int `json:"failing,omitempty"` Why string `json:"why,omitempty"` + // Waits is what the last look of a tool check said the resource waits for (ADR 0283), nil when it said + // healthy or unhealthy; Why then is the answer's own words. + Waits []health.Wait `json:"waits,omitempty"` // probing says a look of the engine's own is under way; due when the next is. probing bool due time.Time @@ -319,7 +328,7 @@ func (j *Judge) Set(resources []Resource) { } else if ok && !sameCheck(k.Check, r.Check) { // Its check changed — another kind, another port, another timing: what the old one found // says nothing about the new, which looks again at once. - k.Check, k.Passed, k.Failing, k.Why, k.due = r.Check, false, 0, "", time.Time{} + k.Check, k.Passed, k.Failing, k.Why, k.Waits, k.due = r.Check, false, 0, "", nil, time.Time{} } } for id := range j.f.Kept { @@ -358,7 +367,7 @@ func (j *Judge) Look(ctx context.Context) (Statement, bool) { j.judge(k, o, now, r.Kind == KindContainer && held[r.Target], blind, why) seen[r.ID] = true st.Resources = append(st.Resources, State{Resource: k.Resource, State: k.State, Reason: k.Reason, Since: k.Since, - Streak: k.Streak, Restarts: k.Counted}) + Streak: k.Streak, Restarts: k.Counted, Waits: waitsOf(k)}) word := k.State + "/" + k.Reason if j.said[r.ID] != word { changed = true @@ -392,7 +401,7 @@ func (j *Judge) judge(k *kept, o Observed, now time.Time, held, blind bool, why k.Seen, k.Identity, k.RuntimeRestarts, k.RuntimeStarted = o.Found, o.Identity, o.Restarts, o.Started k.Started, k.Recent = started, nil // Every start is judged ready afresh (to-be 48 §4). - k.Passed, k.Failing, k.Why, k.due = false, 0, "", time.Time{} + k.Passed, k.Failing, k.Why, k.Waits, k.due = false, 0, "", nil, time.Time{} } // **Held is neither alive nor dead**, and the window ending is a start: judged from a fresh grace. @@ -562,3 +571,11 @@ func firstLine(s string) string { line, _, _ := strings.Cut(strings.TrimSpace(s), "\n") return line } + +// waitsOf is what a resource stated Waiting waits for, and nothing for any other state. +func waitsOf(k *kept) []health.Wait { + if k.State != Waiting { + return nil + } + return append([]health.Wait(nil), k.Waits...) +} diff --git a/internal/liveness/readiness.go b/internal/liveness/readiness.go index a5e1c08..2b37b5d 100644 --- a/internal/liveness/readiness.go +++ b/internal/liveness/readiness.go @@ -14,6 +14,7 @@ import ( "strings" "time" + "git.novox.be/novox/mesh-sdk/go/health" "github.com/novox/mesh-host/internal/declaration" ) @@ -77,6 +78,10 @@ func ready(k *kept, o Observed, inGrace bool) (string, string) { } } switch { + case len(k.Waits) > 0: + // **A wait for the operator is a statement, not a flake** (novox/hq ADR 0283): one look that says it + // is enough, inside the grace too; the next look that says anything else replaces it. + return Waiting, "waits for the operator: " + orSaid(k.Why, "it says what it waits for") case !inGrace && k.Failing >= c.Looks: return Unhealthy, c.Words() + ": " + orSaid(k.Why, "it failed") case k.Passed: @@ -156,8 +161,8 @@ func (j *Judge) Probe(ctx context.Context) { } go func(d dueLook) { defer func() { <-slots }() - ok, why := j.probes().Look(ctx, d.module, d.check) - j.Record(d.id, d.started, ok, why) + ok, why, waits := j.probes().Look(ctx, d.module, d.check) + j.Record(d.id, d.started, ok, why, waits...) }(d) } } @@ -190,9 +195,10 @@ func (j *Judge) due() []dueLook { } // Record keeps what one of the engine's own looks found, for the next look to fold in: a pass is a pass -// whenever it came; a failure counts only after the grace. A look begun before the resource's current -// start says nothing about it. -func (j *Judge) Record(id string, started time.Time, ok bool, why string) { +// whenever it came; a failure counts only after the grace; a look that says the resource waits for the +// operator (waits, ADR 0283) is kept as that, in the grace too, and counts as no failing look. A look begun +// before the resource's current start says nothing about it. +func (j *Judge) Record(id string, started time.Time, ok bool, why string, waits ...health.Wait) { j.mu.Lock() defer j.mu.Unlock() k := j.f.Kept[id] @@ -203,9 +209,12 @@ func (j *Judge) Record(id string, started time.Time, ok bool, why string) { if !k.Started.Equal(started) || k.Check == nil { return } + k.Waits = nil switch { case ok: k.Passed, k.Failing, k.Why = true, 0, "" + case len(waits) > 0: + k.Failing, k.Why, k.Waits = 0, why, append([]health.Wait(nil), waits...) case j.Now().Before(k.Started.Add(j.graceOf(k.Resource))): k.Why = why default: @@ -227,11 +236,12 @@ type Probes struct { // Host is where an endpoint's port is dialled: this machine, by default its loopback. Host string // AskTool asks a module's tool on this machine's own node tools; nil says it cannot be asked. - AskTool func(ctx context.Context, module, tool string) (healthy bool, why string, err error) + AskTool func(ctx context.Context, module, tool string) (health.Answer, error) } -// Look makes one look of a check, and says whether it passed and, when not, what it found. -func (p *Probes) Look(ctx context.Context, module string, c *declaration.Health) (bool, string) { +// Look makes one look of a check, and says whether it passed and, when not, what it found — and, for a +// tool that answers it waits for the operator (ADR 0283), what it waits for. Only a tool check can wait. +func (p *Probes) Look(ctx context.Context, module string, c *declaration.Health) (bool, string, []health.Wait) { timeout := c.Within() if timeout <= 0 { timeout = 5 * time.Second @@ -240,30 +250,33 @@ func (p *Probes) Look(ctx context.Context, module string, c *declaration.Health) defer cancel() switch c.Kind { case declaration.HealthHTTP: - return p.http(ctx, c, timeout) + ok, why := p.http(ctx, c, timeout) + return ok, why, nil case declaration.HealthTCP: conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", p.address(c.Port)) if err != nil { - return false, dialWords(err, timeout) + return false, dialWords(err, timeout), nil } _ = conn.Close() - return true, "" + return true, "", nil case declaration.HealthTool: if p.AskTool == nil { - return false, "its tool cannot be asked: this engine has no link to the node tools" + return false, "its tool cannot be asked: this engine has no link to the node tools", nil } - healthy, why, err := p.AskTool(ctx, module, c.Tool) + a, err := p.AskTool(ctx, module, c.Tool) switch { case errors.Is(err, context.DeadlineExceeded): - return false, fmt.Sprintf("no answer within %s", timeout) + return false, fmt.Sprintf("no answer within %s", timeout), nil case err != nil: - return false, firstLine(err.Error()) - case !healthy: - return false, orSaid(why, "it answered not healthy, and not why") + return false, firstLine(err.Error()), nil + case a.State == health.StateWaits && len(a.Waits) > 0: + return false, orSaid(a.Why, "it waits for the operator, and not why"), a.Waits + case !a.Healthy: + return false, orSaid(a.Why, "it answered not healthy, and not why"), nil } - return true, "" + return true, "", nil } - return false, "the engine does not make a " + c.Kind + " look itself" + return false, "the engine does not make a " + c.Kind + " look itself", nil } func (p *Probes) address(port int) string { diff --git a/internal/liveness/readiness_test.go b/internal/liveness/readiness_test.go index 0c39419..dc0ca3c 100644 --- a/internal/liveness/readiness_test.go +++ b/internal/liveness/readiness_test.go @@ -13,6 +13,7 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-sdk/go/health" "github.com/novox/mesh-host/internal/declaration" ) @@ -40,8 +41,8 @@ func portOf(t *testing.T, url string) int { func lookOnce(t *testing.T, j *Judge) { t.Helper() for _, d := range j.due() { - ok, why := j.probes().Look(t.Context(), d.module, d.check) - j.Record(d.id, d.started, ok, why) + ok, why, waits := j.probes().Look(t.Context(), d.module, d.check) + j.Record(d.id, d.started, ok, why, waits...) } } @@ -182,11 +183,11 @@ func TestNeverMoreLooksThanTheBudget(t *testing.T) { func TestAToolCheckIsAskedOfTheNodeTools(t *testing.T) { asked := "" - p := &Probes{AskTool: func(_ context.Context, module, tool string) (bool, string, error) { + p := &Probes{AskTool: func(_ context.Context, module, tool string) (health.Answer, error) { asked = module + "." + tool - return false, "the admin refused the minted secret", nil + return health.Unhealthy("the admin refused the minted secret"), nil }} - ok, why := p.Look(t.Context(), "keycloak", &declaration.Health{Kind: declaration.HealthTool, Tool: "keycloak_admin_health", + ok, why, _ := p.Look(t.Context(), "keycloak", &declaration.Health{Kind: declaration.HealthTool, Tool: "keycloak_admin_health", Interval: "30s", Timeout: "5s", Looks: 2, Grace: "0s"}) if ok || asked != "keycloak.keycloak_admin_health" || !strings.Contains(why, "refused the minted secret") { t.Fatalf("asked %q: %v %q", asked, ok, why) @@ -198,16 +199,16 @@ func TestAToolCheckIsAskedOfTheNodeTools(t *testing.T) { func TestAToolCheckThatGetsNoAnswerSaysHowLongItWaited(t *testing.T) { check := &declaration.Health{Kind: declaration.HealthTool, Tool: "sensors_health", Interval: "30s", Timeout: "5s", Looks: 2, Grace: "0s"} - p := &Probes{AskTool: func(context.Context, string, string) (bool, string, error) { - return false, "", fmt.Errorf("no answer from mesh.mod.sensors.tool.sensors_health.anchor: %w", context.DeadlineExceeded) + p := &Probes{AskTool: func(context.Context, string, string) (health.Answer, error) { + return health.Answer{}, fmt.Errorf("no answer from mesh.mod.sensors.tool.sensors_health.anchor: %w", context.DeadlineExceeded) }} - if ok, why := p.Look(t.Context(), "sensors", check); ok || why != "no answer within 5s" { + if ok, why, _ := p.Look(t.Context(), "sensors", check); ok || why != "no answer within 5s" { t.Fatalf("%v %q", ok, why) } - p.AskTool = func(context.Context, string, string) (bool, string, error) { - return false, "", errors.New("no link to the bus is open") + p.AskTool = func(context.Context, string, string) (health.Answer, error) { + return health.Answer{}, errors.New("no link to the bus is open") } - if ok, why := p.Look(t.Context(), "sensors", check); ok || why != "no link to the bus is open" { + if ok, why, _ := p.Look(t.Context(), "sensors", check); ok || why != "no link to the bus is open" { t.Fatalf("%v %q", ok, why) } } diff --git a/internal/liveness/replay_test.go b/internal/liveness/replay_test.go index 39c438f..a5ab295 100644 --- a/internal/liveness/replay_test.go +++ b/internal/liveness/replay_test.go @@ -168,8 +168,8 @@ func TestReplaySilentWebAppIsSaidUnhealthy(t *testing.T) { looks := 0 for time.Since(began) < 2*time.Minute { for _, d := range j.due() { - ok, why := j.probes().Look(t.Context(), d.module, d.check) - j.Record(d.id, d.started, ok, why) + ok, why, waits := j.probes().Look(t.Context(), d.module, d.check) + j.Record(d.id, d.started, ok, why, waits...) looks++ } st, _ = j.Look(t.Context()) @@ -183,7 +183,7 @@ func TestReplaySilentWebAppIsSaidUnhealthy(t *testing.T) { t.Fatalf("a web application answering nothing was not said unhealthy within two looks (%d looks): %+v", looks, s) } // And a TCP check would have said it reachable: the port is open. - if ok, why := tcp.Look(t.Context(), "app", timing(&declaration.Health{Kind: declaration.HealthTCP, Port: published})); !ok { + if ok, why, _ := tcp.Look(t.Context(), "app", timing(&declaration.Health{Kind: declaration.HealthTCP, Port: published})); !ok { t.Fatalf("a tcp check could not connect to the silent application: %s", why) } t.Logf("said %s (%s) after %d looks, %s after the judging began; a TCP check connects", s.State, s.Reason, looks, diff --git a/internal/liveness/waiting_test.go b/internal/liveness/waiting_test.go new file mode 100644 index 0000000..cfaf403 --- /dev/null +++ b/internal/liveness/waiting_test.go @@ -0,0 +1,122 @@ +package liveness + +import ( + "context" + "encoding/json" + "path/filepath" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/health" + "github.com/novox/mesh-host/internal/declaration" +) + +// A module's tool check may say it waits for the operator (novox/hq ADR 0283, issue 386): a named secret or +// setting not given yet, and nothing else wrong. The engine states that resource waiting, with its waits, on +// one look, inside the grace too; the next look that says anything else replaces it. + +var password = health.Wait{Part: "the source games", Secret: "smb-password-games", What: "the password of the source games"} + +// toolJudge is a judge of one running process whose tool check answers what *answer holds. +func toolJudge(t *testing.T, answer *[]byte) (*Judge, *clock) { + t.Helper() + c := &clock{now: t0} + rt := &fakeRuntime{containers: map[string]Observed{"watch": running("c1", 0, t0.Format(time.RFC3339Nano))}} + j := aJudge(t, rt, c, filepath.Join(t.TempDir(), FileName)) + j.Probes = &Probes{AskTool: func(context.Context, string, string) (health.Answer, error) { + return health.Read(*answer) + }} + check := &declaration.Health{Kind: declaration.HealthTool, Tool: "mounts_health", Interval: "30s", Timeout: "5s", + Looks: 2, Grace: "1m30s"} + j.Set([]Resource{{Module: "mounts", ID: "mounts.watch", Kind: KindContainer, Target: "watch", Check: check}}) + j.Look(t.Context()) + return j, c +} + +func answerOf(t *testing.T, a health.Answer) []byte { + t.Helper() + b, err := json.Marshal(a) + if err != nil { + t.Fatal(err) + } + return b +} + +func next(t *testing.T, j *Judge, c *clock, by time.Duration) State { + t.Helper() + c.now = c.now.Add(by) + lookOnce(t, j) + st, _ := j.Look(t.Context()) + return stateOf(t, st, "mounts.watch") +} + +func TestAToolThatWaitsIsStatedWaitingWithItsWaitsInsideTheGrace(t *testing.T) { + answer := answerOf(t, health.Waiting("the source games waits for its password", password)) + j, c := toolJudge(t, &answer) + s := next(t, j, c, 0) + if s.State != Waiting || len(s.Waits) != 1 || s.Waits[0].Secret != "smb-password-games" || + !strings.Contains(s.Reason, "waits for the operator: the source games waits for its password") { + t.Fatalf("in its grace, a tool that waits: %+v", s) + } + if s.Streak != 0 { + t.Fatalf("a wait counted as a failing streak: %+v", s) + } +} + +func TestAToolThatWaitsStaysWaitingAfterTheGraceAndIsNoFailingLook(t *testing.T) { + answer := answerOf(t, health.Waiting("waits", password)) + j, c := toolJudge(t, &answer) + for i := 0; i < 5; i++ { + if s := next(t, j, c, 30*time.Second); s.State != Waiting { + t.Fatalf("look %d after the grace: %+v", i, s) + } + } +} + +func TestTheNextHealthyAnswerAfterAWaitIsHealthy(t *testing.T) { + answer := answerOf(t, health.Waiting("waits", password)) + j, c := toolJudge(t, &answer) + next(t, j, c, 2*time.Minute) + answer = answerOf(t, health.Healthy("every source armed")) + if s := next(t, j, c, 30*time.Second); s.State != Healthy || len(s.Waits) != 0 { + t.Fatalf("healthy after a wait: %+v", s) + } +} + +func TestAnUnhealthyAnswerAfterAWaitFollowsTheFailingLooks(t *testing.T) { + answer := answerOf(t, health.Waiting("waits", password)) + j, c := toolJudge(t, &answer) + next(t, j, c, 2*time.Minute) + answer = answerOf(t, health.Unhealthy("the share is unreachable")) + s := next(t, j, c, 30*time.Second) + if s.State == Unhealthy || s.State == Waiting || len(s.Waits) != 0 { + t.Fatalf("one failing look after a wait made it %+v", s) + } + if s = next(t, j, c, 30*time.Second); s.State != Unhealthy || !strings.Contains(s.Reason, "the share is unreachable") { + t.Fatalf("two failing looks after a wait: %+v", s) + } +} + +func TestAnAnswerWithoutStateIsJudgedAsBefore(t *testing.T) { + answer := []byte(`{"healthy":false,"why":"down"}`) + j, c := toolJudge(t, &answer) + next(t, j, c, 2*time.Minute) + if s := next(t, j, c, 30*time.Second); s.State != Unhealthy || len(s.Waits) != 0 { + t.Fatalf("an old unhealthy answer: %+v", s) + } + answer = []byte(`{"healthy":true,"why":"ok"}`) + if s := next(t, j, c, 30*time.Second); s.State != Healthy { + t.Fatalf("an old healthy answer: %+v", s) + } +} + +func TestAMalformedWaitIsUnhealthyAndExcusesNothing(t *testing.T) { + answer := []byte(`{"state":"waits","healthy":false,"why":"it waits","waits":[{"part":"p","what":"the password"}]}`) + j, c := toolJudge(t, &answer) + next(t, j, c, 2*time.Minute) + s := next(t, j, c, 30*time.Second) + if s.State != Unhealthy || len(s.Waits) != 0 || !strings.Contains(s.Reason, "neither a secret nor a setting") { + t.Fatalf("a wait naming nothing: %+v", s) + } +} From 9545121ec93b57774eeeb8f4475a6335b16f434d Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 10 Oct 2026 21:14:14 +0200 Subject: [PATCH 2/2] Clear an old pass on a wait, and say a change of waits (review of #78) A resource that passed, then waited, then failed once after its grace was still stated healthy from the old pass; and a change of what it waits for alone raised no change event, so the controller could miss it. --- internal/liveness/liveness.go | 12 +++- internal/liveness/readiness.go | 3 +- internal/liveness/waiting_test.go | 105 ++++++++++++++++++++++++++++-- 3 files changed, 112 insertions(+), 8 deletions(-) diff --git a/internal/liveness/liveness.go b/internal/liveness/liveness.go index 140d64a..0a5bc94 100644 --- a/internal/liveness/liveness.go +++ b/internal/liveness/liveness.go @@ -368,7 +368,8 @@ func (j *Judge) Look(ctx context.Context) (Statement, bool) { seen[r.ID] = true st.Resources = append(st.Resources, State{Resource: k.Resource, State: k.State, Reason: k.Reason, Since: k.Since, Streak: k.Streak, Restarts: k.Counted, Waits: waitsOf(k)}) - word := k.State + "/" + k.Reason + // What it waits for is part of what is said: a change of waits alone is a change. + word := k.State + "/" + k.Reason + "/" + waitsWord(waitsOf(k)) if j.said[r.ID] != word { changed = true j.said[r.ID] = word @@ -579,3 +580,12 @@ func waitsOf(k *kept) []health.Wait { } return append([]health.Wait(nil), k.Waits...) } + +// waitsWord is a resource's waits in one comparable word. +func waitsWord(ws []health.Wait) string { + var b strings.Builder + for _, w := range ws { + b.WriteString(w.Part + "|" + w.Secret + "|" + w.Setting + "|" + w.What + ";") + } + return b.String() +} diff --git a/internal/liveness/readiness.go b/internal/liveness/readiness.go index 2b37b5d..d8f62b6 100644 --- a/internal/liveness/readiness.go +++ b/internal/liveness/readiness.go @@ -214,7 +214,8 @@ func (j *Judge) Record(id string, started time.Time, ok bool, why string, waits case ok: k.Passed, k.Failing, k.Why = true, 0, "" case len(waits) > 0: - k.Failing, k.Why, k.Waits = 0, why, append([]health.Wait(nil), waits...) + // A pass from before the wait says nothing now: a failing look after it counts from no pass. + k.Passed, k.Failing, k.Why, k.Waits = false, 0, why, append([]health.Wait(nil), waits...) case j.Now().Before(k.Started.Add(j.graceOf(k.Resource))): k.Why = why default: diff --git a/internal/liveness/waiting_test.go b/internal/liveness/waiting_test.go index cfaf403..df367ba 100644 --- a/internal/liveness/waiting_test.go +++ b/internal/liveness/waiting_test.go @@ -18,20 +18,27 @@ import ( var password = health.Wait{Part: "the source games", Secret: "smb-password-games", What: "the password of the source games"} +var toolCheck = &declaration.Health{Kind: declaration.HealthTool, Tool: "mounts_health", Interval: "30s", Timeout: "5s", + Looks: 2, Grace: "1m30s"} + // toolJudge is a judge of one running process whose tool check answers what *answer holds. func toolJudge(t *testing.T, answer *[]byte) (*Judge, *clock) { + j, c, _, _ := toolJudgeAt(t, answer, filepath.Join(t.TempDir(), FileName)) + return j, c +} + +// toolJudgeAt is toolJudge keeping its file at path, with its runtime. +func toolJudgeAt(t *testing.T, answer *[]byte, path string) (*Judge, *clock, *fakeRuntime, string) { t.Helper() c := &clock{now: t0} rt := &fakeRuntime{containers: map[string]Observed{"watch": running("c1", 0, t0.Format(time.RFC3339Nano))}} - j := aJudge(t, rt, c, filepath.Join(t.TempDir(), FileName)) + j := aJudge(t, rt, c, path) j.Probes = &Probes{AskTool: func(context.Context, string, string) (health.Answer, error) { return health.Read(*answer) }} - check := &declaration.Health{Kind: declaration.HealthTool, Tool: "mounts_health", Interval: "30s", Timeout: "5s", - Looks: 2, Grace: "1m30s"} - j.Set([]Resource{{Module: "mounts", ID: "mounts.watch", Kind: KindContainer, Target: "watch", Check: check}}) + j.Set([]Resource{{Module: "mounts", ID: "mounts.watch", Kind: KindContainer, Target: "watch", Check: toolCheck}}) j.Look(t.Context()) - return j, c + return j, c, rt, path } func answerOf(t *testing.T, a health.Answer) []byte { @@ -90,7 +97,7 @@ func TestAnUnhealthyAnswerAfterAWaitFollowsTheFailingLooks(t *testing.T) { next(t, j, c, 2*time.Minute) answer = answerOf(t, health.Unhealthy("the share is unreachable")) s := next(t, j, c, 30*time.Second) - if s.State == Unhealthy || s.State == Waiting || len(s.Waits) != 0 { + if s.State == Unhealthy || s.State == Waiting || s.State == Healthy || len(s.Waits) != 0 { t.Fatalf("one failing look after a wait made it %+v", s) } if s = next(t, j, c, 30*time.Second); s.State != Unhealthy || !strings.Contains(s.Reason, "the share is unreachable") { @@ -120,3 +127,89 @@ func TestAMalformedWaitIsUnhealthyAndExcusesNothing(t *testing.T) { t.Fatalf("a wait naming nothing: %+v", s) } } + +// A pass from before a wait is not kept through it: passed, waited, then one failing look after the grace is +// not healthy. +func TestAPassBeforeAWaitIsNotKeptThroughIt(t *testing.T) { + answer := answerOf(t, health.Healthy("ok")) + j, c := toolJudge(t, &answer) + if s := next(t, j, c, 0); s.State != Healthy { + t.Fatalf("passing: %+v", s) + } + answer = answerOf(t, health.Waiting("waits", password)) + next(t, j, c, 2*time.Minute) + answer = answerOf(t, health.Unhealthy("the share is unreachable")) + if s := next(t, j, c, 30*time.Second); s.State == Healthy || s.State == Waiting { + t.Fatalf("one failing look after a wait that followed a pass: %+v", s) + } +} + +// A resource that is down is down, whatever its last look said it waits for. +func TestDownBeatsAWait(t *testing.T) { + answer := answerOf(t, health.Waiting("waits", password)) + j, c, rt, _ := toolJudgeAt(t, &answer, filepath.Join(t.TempDir(), FileName)) + next(t, j, c, 2*time.Minute) + rt.containers["watch"] = Observed{Found: true, Identity: "c1", Started: t0.Format(time.RFC3339Nano)} + c.now = c.now.Add(15 * time.Second) + st, _ := j.Look(t.Context()) + if s := stateOf(t, st, "mounts.watch"); s.State != Unhealthy || s.Reason != ReasonDown || len(s.Waits) != 0 { + t.Fatalf("a stopped process that waited: %+v", s) + } +} + +// A changed check says nothing of what the old one found waiting. +func TestAChangedCheckClearsTheWaits(t *testing.T) { + answer := answerOf(t, health.Waiting("waits", password)) + j, c := toolJudge(t, &answer) + next(t, j, c, 2*time.Minute) + other := *toolCheck + other.Interval = "60s" + j.Set([]Resource{{Module: "mounts", ID: "mounts.watch", Kind: KindContainer, Target: "watch", Check: &other}}) + st, _ := j.Look(t.Context()) + if s := stateOf(t, st, "mounts.watch"); s.State == Waiting || len(s.Waits) != 0 { + t.Fatalf("after its check changed: %+v", s) + } +} + +// The engine restarted keeps what a resource waits for, as it keeps everything else it judged. +func TestWaitsAreKeptAcrossTheEnginesRestart(t *testing.T) { + answer := answerOf(t, health.Waiting("waits", password)) + path := filepath.Join(t.TempDir(), FileName) + j, c, rt, _ := toolJudgeAt(t, &answer, path) + next(t, j, c, 2*time.Minute) + again := aJudge(t, rt, c, path) + again.Set([]Resource{{Module: "mounts", ID: "mounts.watch", Kind: KindContainer, Target: "watch", Check: toolCheck}}) + st, _ := again.Look(t.Context()) + if s := stateOf(t, st, "mounts.watch"); s.State != Waiting || len(s.Waits) != 1 || s.Waits[0].Secret != password.Secret { + t.Fatalf("after the engine restarted: %+v", s) + } +} + +// Waiting and healthy each way is a change, and so is a change of what it waits for alone. +func TestAChangeOfWaitingOrOfItsWaitsIsAChange(t *testing.T) { + answer := answerOf(t, health.Healthy("ok")) + j, c := toolJudge(t, &answer) + look := func(by time.Duration) bool { + c.now = c.now.Add(by) + lookOnce(t, j) + _, changed := j.Look(t.Context()) + return changed + } + look(0) + answer = answerOf(t, health.Waiting("waits", password)) + if !look(30 * time.Second) { + t.Fatal("healthy to waiting was no change") + } + if look(30 * time.Second) { + t.Fatal("the same wait again was a change") + } + user := health.Wait{Part: "the source games", Setting: "smb-users", What: "the username of the source games"} + answer = answerOf(t, health.Waiting("waits", password, user)) + if !look(30 * time.Second) { + t.Fatal("a wait added was no change") + } + answer = answerOf(t, health.Healthy("ok")) + if !look(30 * time.Second) { + t.Fatal("waiting to healthy was no change") + } +}