jschoubben
4b2d5aeaab
It reads the keeper's status from its own state asked, what the keeper holds, and how many accepted verdicts wait on its consumer <node>_nats_keeper on EVENTS from the monitoring endpoint, and answers HealthOf's judgement: unhealthy, naming why, past the three-minute bound; the entries refused, waiting and past the raise mark named beside. Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig
90 lines
3.1 KiB
Go
90 lines
3.1 KiB
Go
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"strings"
|
|
"time"
|
|
|
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
|
"git.novox.be/novox/mesh-sdk/go/health"
|
|
|
|
"nats-tools/internal/keeper"
|
|
)
|
|
|
|
// The keeper's health, as the node-engine asks it of the process keeper (design 25 § 12.2-12.3; CAT-N6): its status
|
|
// from its own state asked, what it holds, and how many accepted verdicts wait on its consumer, read from the bus's
|
|
// monitoring endpoint.
|
|
|
|
// StatusInState reads the keeper's status through the runtime.
|
|
func StatusInState() (keeper.Status, error) {
|
|
e, err := stdio.State(keeper.StateAsked).Get(keeper.KeyStatus)
|
|
if err != nil || e == nil {
|
|
return keeper.Status{}, err
|
|
}
|
|
var s keeper.Status
|
|
return s, json.Unmarshal(e.Value, &s)
|
|
}
|
|
|
|
// keeperPending is how many messages wait on the keeper's consumer on EVENTS (`<node>_nats_keeper`), or -1 unread.
|
|
func keeperPending(jsz []byte) int {
|
|
var doc struct {
|
|
Accounts []struct {
|
|
Streams []struct {
|
|
Name string `json:"name"`
|
|
Consumers []struct {
|
|
Name string `json:"name"`
|
|
Pending int `json:"num_pending"`
|
|
} `json:"consumer_detail"`
|
|
} `json:"stream_detail"`
|
|
} `json:"account_details"`
|
|
}
|
|
if json.Unmarshal(jsz, &doc) != nil {
|
|
return -1
|
|
}
|
|
for _, a := range doc.Accounts {
|
|
for _, s := range a.Streams {
|
|
for _, c := range s.Consumers {
|
|
if s.Name == "EVENTS" && strings.HasSuffix(c.Name, "_"+keeper.Module+"_keeper") {
|
|
return c.Pending
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return -1
|
|
}
|
|
|
|
// keeperHealth is the tool's answer.
|
|
func keeperHealth(status func() (keeper.Status, error), read heldEntries, jsz func() ([]byte, error), now func() time.Time) health.Answer {
|
|
s, err := status()
|
|
if err != nil {
|
|
return health.Unhealthy("The keeper's status cannot be read from its own state: " + err.Error())
|
|
}
|
|
held, _ := read()
|
|
pending := -1
|
|
if doc, err := jsz(); err == nil {
|
|
pending = keeperPending(doc)
|
|
}
|
|
h := keeper.HealthOf(s, held, pending, now())
|
|
why := h.Why + "; last pass " + h.Pass
|
|
for _, l := range []struct {
|
|
name string
|
|
lines []string
|
|
}{{"refused", h.Refused}, {"waiting", h.Waiting}, {"past the raise mark", h.Full}} {
|
|
if len(l.lines) > 0 {
|
|
why += "; " + l.name + ": " + strings.Join(l.lines, ", ")
|
|
}
|
|
}
|
|
if !h.Healthy {
|
|
return health.Unhealthy("The keeper is unhealthy: " + why + ".")
|
|
}
|
|
return health.Healthy("The keeper is healthy: " + why + ".")
|
|
}
|
|
|
|
func keeperHealthTool(status func() (keeper.Status, error), read heldEntries, jsz func() ([]byte, error)) stdio.Tool {
|
|
return stdio.Tool{Name: "nats_keeper_health",
|
|
Description: "The bus module's keeper's health, as the node-engine asks it: unhealthy, naming why, when its fill pass is more than three minutes old, its first view (the indexer's indexer_bus_state) has not answered, or accepted verdicts wait on its consumer unanswered past three minutes; it names beside the entries it could not create, the ones waiting, and those past the raise mark (another module's full entry raises that module's condition and leaves the keeper healthy).",
|
|
Run: func(map[string]any) (any, error) {
|
|
return keeperHealth(status, read, jsz, func() time.Time { return time.Now().UTC() }), nil
|
|
}}
|
|
}
|