mesh/delivery closed: a newer head of the same pull request
mesh/check failed (60): integration: mesh-knowledge/cmd/mesh-knowledge.TestARecordThatNeedsTheOperatorIsAc…; and 59 more | 7c78fd2e0c4f on f469eb387a30
A pass runs on every change of its node, and a newly placed module stays unpublished for the minutes its build takes, so counting passes made every new placement unhealthy while nothing was wrong. The health now says a hold only once a module has stayed unpublished at one build for longer than 45 minutes (the build step's bound is 30), with since when; a published module or another build starts the clock again. Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
158 lines
6.6 KiB
Go
158 lines
6.6 KiB
Go
package main
|
|
|
|
// The holder's tools (novox/hq ADR 0570 Consequences, ADR 0564 §12's rule): a read of every key of `provisions` and
|
|
// `plans`, one key whole, each state's fill, making one node's plans now (retiring more at once on the operator's
|
|
// word), reading the operator's answers now, and its health: a retirement held back as a fault of the mesh (issue 940).
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"strings"
|
|
|
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
|
"git.novox.be/novox/mesh-sdk/go/health"
|
|
"git.novox.be/novox/mesh-sdk/go/provisions"
|
|
)
|
|
|
|
// The holder's tools.
|
|
const (
|
|
ProvisionsTool = "provisioner_provisions"
|
|
PlansTool = "provisioner_plans"
|
|
ComposeTool = "provisioner_compose"
|
|
DecideTool = "provisioner_decide"
|
|
HealthTool = "provisioner_health"
|
|
)
|
|
|
|
func str(d string) map[string]any { return map[string]any{"type": "string", "description": d} }
|
|
|
|
func holderTools(h *Holder) []stdio.Tool {
|
|
arg := func(a map[string]any, k string) string { s, _ := a[k].(string); return strings.TrimSpace(s) }
|
|
return []stdio.Tool{
|
|
{Name: ProvisionsTool,
|
|
Description: "Every provision the seat keeps (novox/hq ADR 0570): its key <node>.<consumer>.<provision>[.<local>], " +
|
|
"provider, consumer and state (asked, provided, waits, failed, retiring, with why), narrowed by node and " +
|
|
"state, with how many keys and bytes the state holds; or one key whole.",
|
|
Input: map[string]any{"type": "object", "properties": map[string]any{"key": str("one key, whole"),
|
|
"node": str("a consumer's node"), "state": str("asked, provided, waits, failed or retiring")}},
|
|
Run: func(a map[string]any) (any, error) {
|
|
return h.read(ProvisionsState, arg(a, "key"), arg(a, "node"), arg(a, "state"))
|
|
}},
|
|
{Name: PlansTool,
|
|
Description: "Every module's provisioning plan the seat keeps, <node>.<module>, as the node-engine's published " +
|
|
"composition carries it (composed-<node>), narrowed by node, with how many keys and bytes the state holds, " +
|
|
"and each placed module held with its last plan kept and why (refused: the node-engine refused its " +
|
|
"composition, or has not published it at the wanted build yet); or one plan whole.",
|
|
Input: map[string]any{"type": "object", "properties": map[string]any{"key": str("one plan, <node>.<module>"),
|
|
"node": str("a node")}},
|
|
Run: func(a map[string]any) (any, error) {
|
|
out, err := h.read(PlansState, arg(a, "key"), arg(a, "node"), "")
|
|
if err != nil || arg(a, "key") != "" {
|
|
return out, err
|
|
}
|
|
refused, err := h.read(RefusedState, "", arg(a, "node"), "")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out.(map[string]any)[RefusedState] = refused.(map[string]any)[RefusedState]
|
|
return out, nil
|
|
}},
|
|
{Name: ComposeTool,
|
|
Description: "Make one node's plans and provisions now from the node-engine's published compositions and the " +
|
|
"providers' outcomes, as a change event would (it composes nothing), and say what was written, removed, " +
|
|
"changed and asked, and where the time went. A pass that would retire the provisions of most of the " +
|
|
"node's consumers at once retires none and says it as a fault (provisioner_health); retire-many: true " +
|
|
"is the operator's word that such a removal is meant, and retires them in this pass, each asked.",
|
|
Input: map[string]any{"type": "object", "properties": map[string]any{"node": str("the node"),
|
|
"retire-many": map[string]any{"type": "boolean", "description": "retire past the bound in this pass"}},
|
|
"required": []string{"node"}},
|
|
Run: func(a map[string]any) (any, error) {
|
|
if arg(a, "node") == "" {
|
|
return nil, fmt.Errorf("compose names the node")
|
|
}
|
|
many, _ := a["retire-many"].(bool)
|
|
return h.ComposeNodeWith(context.Background(), arg(a, "node"), many)
|
|
}},
|
|
{Name: DecideTool,
|
|
Description: "Read the operator's answers to every retiring provision's ask now, writing each warrant into its " +
|
|
"key, as the router's word on an ask would.",
|
|
Run: func(map[string]any) (any, error) { return map[string]any{"said": h.Decide()}, nil }},
|
|
{Name: HealthTool,
|
|
Description: "The provisioner's runtime health, as the node-engine asks it (novox/hq ADR 0366): unhealthy while " +
|
|
"a pass held back a retirement of most of a node's consumers at once as a fault of the mesh (issue 940), " +
|
|
"naming the node, the consumers and the likely cause, or while a module's composition has stayed " +
|
|
"unpublished at one build for longer than 45 minutes (longer than a build takes), naming the module, " +
|
|
"the build and since when.",
|
|
Run: func(map[string]any) (any, error) { return h.health(), nil }},
|
|
}
|
|
}
|
|
|
|
// health is the holder's runtime health: unhealthy while a retirement is held back as a fault of the mesh.
|
|
func (h *Holder) health() health.Answer {
|
|
faults, holds := h.Faults(), h.LastingHolds()
|
|
if len(faults)+len(holds) == 0 {
|
|
return health.Healthy("No pass holds back a retirement of most of a node's consumers, and no module's " +
|
|
"composition has stayed unpublished at one build for longer than " + unpublishedFor.String() + ".")
|
|
}
|
|
var said []string
|
|
for _, n := range sortedKeys(faults) {
|
|
said = append(said, faults[n])
|
|
}
|
|
for _, k := range sortedKeys(holds) {
|
|
said = append(said, k+": "+holds[k]+"; its plan and its provisions are kept as they were.")
|
|
}
|
|
return health.Unhealthy(strings.Join(said, " "))
|
|
}
|
|
|
|
// read is a state's keys narrowed, or one whole, with the state's fill.
|
|
func (h *Holder) read(state, key, node, want string) (any, error) {
|
|
if key != "" {
|
|
e, err := h.Store.Get(state, key)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if e == nil {
|
|
return nil, fmt.Errorf("%s holds no %s", state, key)
|
|
}
|
|
return json.RawMessage(e.Value), nil
|
|
}
|
|
keys, err := h.Store.Keys(state)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
rows := []map[string]any{}
|
|
bytes := 0
|
|
for _, k := range keys {
|
|
e, err := h.Store.Get(state, k)
|
|
if err != nil || e == nil {
|
|
continue
|
|
}
|
|
bytes += len(e.Value)
|
|
if node != "" && !strings.HasPrefix(k, node+".") {
|
|
continue
|
|
}
|
|
row := map[string]any{"key": k}
|
|
if state == ProvisionsState {
|
|
p, err := provisions.Decode(e.Value)
|
|
if err != nil {
|
|
row["unreadable"] = err.Error()
|
|
} else {
|
|
if want != "" && p.State != want {
|
|
continue
|
|
}
|
|
row["state"], row["provider"], row["why"] = p.State, p.Provider.Module+" on "+p.Provider.Node, p.Why
|
|
}
|
|
}
|
|
if state == RefusedState {
|
|
var r Refusal
|
|
if err := json.Unmarshal(e.Value, &r); err != nil {
|
|
row["unreadable"] = err.Error()
|
|
} else {
|
|
row["module"], row["why"], row["since"] = r.Module, r.Why, r.Since
|
|
}
|
|
}
|
|
rows = append(rows, row)
|
|
}
|
|
return map[string]any{state: rows, "count": len(rows), "fill": map[string]int{"keys": len(keys), "bytes": bytes}}, nil
|
|
}
|