H-A-311 (3): runner_health names a running pipeline with nothing open and nothing placed for five minutes
main.137 stood seventeen minutes with nothing open while runner_health said healthy. The tools program says each such pipeline in its journal once per key (H-A-313). Builds: design 47 H H-A-311 (3), H-A-313
This commit is contained in:
@@ -7,6 +7,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
@@ -98,13 +99,29 @@ func FirstViewTools(module string, st stdio.KeptState, acts views.HandActs, now
|
||||
}
|
||||
|
||||
// HealthTool is the runner's health check, run outside its process (design 47 G §76, §468, §495; A-164): it reads the
|
||||
// runner's state pipelines as show does and the ages its process writes in the state consumers, and asks no seat verb.
|
||||
func HealthTool(pipelines, consumers pipeline.Kept, now func() time.Time) stdio.Tool {
|
||||
// runner's state pipelines as show does and the ages its process writes in the state consumers, and asks no seat verb. A
|
||||
// running pipeline it first names stalled is said in this program's journal through logf, once per key for the
|
||||
// program's life (H-A-311 (3), H-A-313).
|
||||
func HealthTool(pipelines, consumers pipeline.Kept, now func() time.Time, logf func(string, ...any)) stdio.Tool {
|
||||
var mu sync.Mutex
|
||||
said := map[string]bool{}
|
||||
return stdio.Tool{Name: Prefix + "_health",
|
||||
Description: "The pipeline runner's health: healthy when it answers show and each of its durable consumers' oldest " +
|
||||
"unacknowledged message waits less than five minutes; otherwise it names each consumer behind.",
|
||||
Description: "The pipeline runner's health: healthy when it answers show, each of its durable consumers' oldest " +
|
||||
"unacknowledged message waits less than five minutes and no running pipeline has had nothing open and nothing " +
|
||||
"newly placed for five minutes; otherwise it names each consumer behind and each such pipeline by its key.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) { return pipeline.Health(pipelines, consumers, now()), nil }}
|
||||
Run: func(map[string]any) (any, error) {
|
||||
a, stalled := pipeline.HealthStalled(pipelines, consumers, now())
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
for _, key := range stalled {
|
||||
if !said[key] && logf != nil {
|
||||
said[key] = true
|
||||
logf("the running pipeline %s has had nothing open and nothing newly placed for %s or more: runner_health names it unhealthy (H-A-311)", key, pipeline.StalledAfter)
|
||||
}
|
||||
}
|
||||
return a, nil
|
||||
}}
|
||||
}
|
||||
|
||||
// StoresFill is each of the runner's stores' fill through the bundle's calls (design 47 G §366, §623, A-214): the states
|
||||
@@ -154,7 +171,8 @@ func main() {
|
||||
tools = append(tools, CancelTool(r.List, stdio.Ask), RerunTool(stdio.Ask))
|
||||
tools = append(tools, VerbTools(stdio.Ask)...)
|
||||
tools = append(tools, TestRunsTools(stdio.Ask)...)
|
||||
tools = append(tools, HealthTool(r.Pipelines, kept{stdio.State(pipeline.StateConsumers)}, time.Now))
|
||||
tools = append(tools, HealthTool(r.Pipelines, kept{stdio.State(pipeline.StateConsumers)}, time.Now,
|
||||
func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] "+f+"\n", a...) }))
|
||||
tools = append(tools, WaitingTool(kept{stdio.State(seat.StateQueue)}))
|
||||
if err := stdio.Serve("", tools); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] %v\n", err)
|
||||
|
||||
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -37,7 +38,7 @@ func TestTools(t *testing.T) {
|
||||
names = append(names, tl.Name)
|
||||
}
|
||||
// RN25 (§468, §76; A-164): the health check is the bundle's runner_health.
|
||||
names = append(names, HealthTool(none{}, none{}, nil).Name, WaitingTool(none{}).Name)
|
||||
names = append(names, HealthTool(none{}, none{}, nil, nil).Name, WaitingTool(none{}).Name)
|
||||
// §606, §549: the unlogged entries are listed and appended by hand.
|
||||
for _, tl := range UnloggedTools(stdio.State("pipelines"), stdio.Log("step-log"), nil, nil) {
|
||||
names = append(names, tl.Name)
|
||||
@@ -53,6 +54,22 @@ func TestTools(t *testing.T) {
|
||||
if got, _ := json.Marshal(out); err != nil || string(got) != `[{"sequence":1,"pipeline":"main.5"},{"sequence":3,"pipeline":"main.7"}]` {
|
||||
t.Errorf("runner_waiting = %s (%v)", got, err)
|
||||
}
|
||||
// H-A-311 (3), H-A-313: runner_health names a running pipeline quiet five minutes, and this program's journal says it
|
||||
// once per key, however often the health is asked.
|
||||
now := time.Date(2026, 11, 21, 20, 35, 0, 0, time.UTC)
|
||||
quiet := queueOf{"main.137": `{"pipeline":"main.137","branch":"main","outcome":"running","source":137,"started":"2026-11-21T20:15:00Z",` +
|
||||
`"movers":[{"module":"app"}],"steps":[{"module":"app","step":"build","outcome":"passed","status":"required","started":"2026-11-21T20:15:52Z","ended":"2026-11-21T20:17:55Z"}]}`}
|
||||
var journal []string
|
||||
health := HealthTool(quiet, none{}, func() time.Time { return now }, func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) })
|
||||
for range 2 {
|
||||
out, err := health.Run(nil)
|
||||
if got, _ := json.Marshal(out); err != nil || !strings.Contains(string(got), "main.137 (nothing open and nothing placed for 17m5s)") {
|
||||
t.Errorf("runner_health = %s (%v), want main.137 named stalled", got, err)
|
||||
}
|
||||
}
|
||||
if len(journal) != 1 || !strings.Contains(journal[0], "main.137") {
|
||||
t.Errorf("the journal said %q, want main.137 said once", journal)
|
||||
}
|
||||
}
|
||||
|
||||
// queueOf is a state queue in memory, its keys in no order.
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
@@ -19,13 +20,78 @@ const StateConsumers = process.ConsumersState
|
||||
// behind (§76).
|
||||
const BehindAfter = 5 * time.Minute
|
||||
|
||||
// Health is the runner's health as its tool runner_health answers it (§468, §76, §495; A-164): healthy when its state
|
||||
// pipelines answers, as show reads it, and each durable consumer's oldest unacknowledged message waits less than
|
||||
// BehindAfter; otherwise unhealthy naming each consumer behind and how long.
|
||||
func Health(pipelines, consumers Kept, now time.Time) health.Answer {
|
||||
if _, err := pipelines.Keys(); err != nil {
|
||||
return health.Unhealthy(fmt.Sprintf("The runner does not answer show: its state pipelines cannot be listed (%v).", err))
|
||||
// StalledAfter is how long a running pipeline may have nothing open and nothing newly placed before the runner's health
|
||||
// names it stalled (H-A-311 (3)).
|
||||
const StalledAfter = 5 * time.Minute
|
||||
|
||||
// Stalled is the key of each running pipeline, as show reads it, that has had nothing open (no step asked and not ended)
|
||||
// and nothing newly placed, started or ended for StalledAfter at now (H-A-311 (3)), each with how long, in key order.
|
||||
func Stalled(pipelines Kept, now time.Time) ([]string, error) {
|
||||
all, err := Reads{Pipelines: pipelines}.pipelines()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []string
|
||||
for _, p := range all {
|
||||
if p.Outcome != seat.PipelineOutcomeRunning {
|
||||
continue
|
||||
}
|
||||
last, busy := startedOf(p), false
|
||||
for _, st := range p.Steps {
|
||||
if open(st) {
|
||||
busy = true
|
||||
break
|
||||
}
|
||||
for _, at := range []*time.Time{st.Started, st.Ended} {
|
||||
if at != nil && at.After(last) {
|
||||
last = *at
|
||||
}
|
||||
}
|
||||
}
|
||||
if busy || last.IsZero() || now.Sub(last) < StalledAfter {
|
||||
continue
|
||||
}
|
||||
out = append(out, fmt.Sprintf("%s (nothing open and nothing placed for %s)", p.Pipeline, now.Sub(last).Round(time.Second)))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Health is the runner's health as its tool runner_health answers it (§468, §76, §495; A-164): healthy when its state
|
||||
// pipelines answers, as show reads it, each durable consumer's oldest unacknowledged message waits less than
|
||||
// BehindAfter, and no running pipeline is stalled (H-A-311 (3)); otherwise unhealthy naming each consumer behind and
|
||||
// how long, and each stalled pipeline by its key.
|
||||
func Health(pipelines, consumers Kept, now time.Time) health.Answer {
|
||||
a, _ := HealthStalled(pipelines, consumers, now)
|
||||
return a
|
||||
}
|
||||
|
||||
// HealthStalled is Health with the keys of the running pipelines it names stalled, so the tools program says each in its
|
||||
// journal once (H-A-313).
|
||||
func HealthStalled(pipelines, consumers Kept, now time.Time) (health.Answer, []string) {
|
||||
if _, err := pipelines.Keys(); err != nil {
|
||||
return health.Unhealthy(fmt.Sprintf("The runner does not answer show: its state pipelines cannot be listed (%v).", err)), nil
|
||||
}
|
||||
stalled, err := Stalled(pipelines, now)
|
||||
if err != nil {
|
||||
return health.Unhealthy(fmt.Sprintf("The runner's running pipelines cannot be read as show reads them (%v).", err)), nil
|
||||
}
|
||||
var keys []string
|
||||
for _, s := range stalled {
|
||||
keys = append(keys, strings.SplitN(s, " ", 2)[0])
|
||||
}
|
||||
a := consumersHealth(consumers, now)
|
||||
if len(stalled) == 0 {
|
||||
return a, nil
|
||||
}
|
||||
why := fmt.Sprintf("The runner's running pipelines with nothing open and nothing newly placed for %s or more: %s.", StalledAfter, strings.Join(stalled, ", "))
|
||||
if !a.Healthy {
|
||||
why = a.Why + " " + why
|
||||
}
|
||||
return health.Unhealthy(why), keys
|
||||
}
|
||||
|
||||
// consumersHealth is the part of Health its consumers' ages give.
|
||||
func consumersHealth(consumers Kept, now time.Time) health.Answer {
|
||||
keys, err := consumers.Keys()
|
||||
if err != nil {
|
||||
return health.Unhealthy(fmt.Sprintf("The ages of the runner's consumers cannot be read from its state %s (%v).", StateConsumers, err))
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
@@ -50,6 +51,11 @@ func TestHealth(t *testing.T) {
|
||||
{"six minutes names the consumer", mapKept{}, age("n_runner", 6*time.Minute), false, "n_runner (its oldest unacknowledged message waits 6m0s)"},
|
||||
{"show does not answer", failingKept{}, age("n_runner", 0), false, "does not answer show"},
|
||||
{"no event heard yet", mapKept{}, mapKept{}, true, "has heard no event yet"},
|
||||
// H-A-311 (3): a running pipeline with nothing open and nothing newly placed for five minutes is named by its key.
|
||||
{"a pipeline quiet six minutes is named", piped(now, "main.137", -6*time.Minute, seat.StepResultOutcomePassed), age("n_runner", 0), false,
|
||||
"main.137 (nothing open and nothing placed for 6m0s)"},
|
||||
{"a pipeline quiet four minutes is not", piped(now, "main.137", -4*time.Minute, seat.StepResultOutcomePassed), age("n_runner", 0), true, ""},
|
||||
{"a pipeline with a step open is not", piped(now, "main.137", -20*time.Minute, seat.StepResultOutcomeRunning), age("n_runner", 0), true, ""},
|
||||
}
|
||||
for _, r := range rows {
|
||||
a := Health(r.pipelines, r.consumers, now)
|
||||
@@ -58,3 +64,16 @@ func TestHealth(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// piped is the state pipelines holding one running pipeline key whose one step was placed and last moved at now+moved,
|
||||
// with outcome.
|
||||
func piped(now time.Time, key string, moved time.Duration, outcome string) mapKept {
|
||||
at, started := now.Add(moved), now.Add(-time.Hour)
|
||||
st := seat.StepResult{Module: "app", Step: seat.StepResultStepBuild, Outcome: outcome, Status: seat.StepResultStatusRequired, Started: &at}
|
||||
if outcome != seat.StepResultOutcomeRunning {
|
||||
st.Ended = &at
|
||||
}
|
||||
raw, _ := json.Marshal(seat.Pipeline{Pipeline: key, Branch: "main", Outcome: seat.PipelineOutcomeRunning, Started: &started,
|
||||
Movers: []seat.Mover{{Module: "app"}}, Steps: []seat.StepResult{st}})
|
||||
return mapKept{key: raw}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user