H-A-311 (3): runner_health names a running pipeline with nothing open and nothing placed for five minutes

main.137 stood seventeen minutes with nothing open while runner_health
said healthy. The tools program says each such pipeline in its journal
once per key (H-A-313).

Builds: design 47 H H-A-311 (3), H-A-313
This commit is contained in:
2026-11-21 20:44:43 +01:00
parent e137df7189
commit f22945c26a
4 changed files with 133 additions and 13 deletions
@@ -7,6 +7,7 @@ import (
"encoding/json"
"fmt"
"os"
"sync"
"time"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
@@ -98,13 +99,29 @@ func FirstViewTools(module string, st stdio.KeptState, acts views.HandActs, now
}
// HealthTool is the runner's health check, run outside its process (design 47 G §76, §468, §495; A-164): it reads the
// runner's state pipelines as show does and the ages its process writes in the state consumers, and asks no seat verb.
func HealthTool(pipelines, consumers pipeline.Kept, now func() time.Time) stdio.Tool {
// runner's state pipelines as show does and the ages its process writes in the state consumers, and asks no seat verb. A
// running pipeline it first names stalled is said in this program's journal through logf, once per key for the
// program's life (H-A-311 (3), H-A-313).
func HealthTool(pipelines, consumers pipeline.Kept, now func() time.Time, logf func(string, ...any)) stdio.Tool {
var mu sync.Mutex
said := map[string]bool{}
return stdio.Tool{Name: Prefix + "_health",
Description: "The pipeline runner's health: healthy when it answers show and each of its durable consumers' oldest " +
"unacknowledged message waits less than five minutes; otherwise it names each consumer behind.",
Description: "The pipeline runner's health: healthy when it answers show, each of its durable consumers' oldest " +
"unacknowledged message waits less than five minutes and no running pipeline has had nothing open and nothing " +
"newly placed for five minutes; otherwise it names each consumer behind and each such pipeline by its key.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) { return pipeline.Health(pipelines, consumers, now()), nil }}
Run: func(map[string]any) (any, error) {
a, stalled := pipeline.HealthStalled(pipelines, consumers, now())
mu.Lock()
defer mu.Unlock()
for _, key := range stalled {
if !said[key] && logf != nil {
said[key] = true
logf("the running pipeline %s has had nothing open and nothing newly placed for %s or more: runner_health names it unhealthy (H-A-311)", key, pipeline.StalledAfter)
}
}
return a, nil
}}
}
// StoresFill is each of the runner's stores' fill through the bundle's calls (design 47 G §366, §623, A-214): the states
@@ -154,7 +171,8 @@ func main() {
tools = append(tools, CancelTool(r.List, stdio.Ask), RerunTool(stdio.Ask))
tools = append(tools, VerbTools(stdio.Ask)...)
tools = append(tools, TestRunsTools(stdio.Ask)...)
tools = append(tools, HealthTool(r.Pipelines, kept{stdio.State(pipeline.StateConsumers)}, time.Now))
tools = append(tools, HealthTool(r.Pipelines, kept{stdio.State(pipeline.StateConsumers)}, time.Now,
func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] "+f+"\n", a...) }))
tools = append(tools, WaitingTool(kept{stdio.State(seat.StateQueue)}))
if err := stdio.Serve("", tools); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] %v\n", err)
@@ -3,6 +3,7 @@ package main
import (
"encoding/json"
"errors"
"fmt"
"strings"
"testing"
@@ -37,7 +38,7 @@ func TestTools(t *testing.T) {
names = append(names, tl.Name)
}
// RN25 (§468, §76; A-164): the health check is the bundle's runner_health.
names = append(names, HealthTool(none{}, none{}, nil).Name, WaitingTool(none{}).Name)
names = append(names, HealthTool(none{}, none{}, nil, nil).Name, WaitingTool(none{}).Name)
// §606, §549: the unlogged entries are listed and appended by hand.
for _, tl := range UnloggedTools(stdio.State("pipelines"), stdio.Log("step-log"), nil, nil) {
names = append(names, tl.Name)
@@ -53,6 +54,22 @@ func TestTools(t *testing.T) {
if got, _ := json.Marshal(out); err != nil || string(got) != `[{"sequence":1,"pipeline":"main.5"},{"sequence":3,"pipeline":"main.7"}]` {
t.Errorf("runner_waiting = %s (%v)", got, err)
}
// H-A-311 (3), H-A-313: runner_health names a running pipeline quiet five minutes, and this program's journal says it
// once per key, however often the health is asked.
now := time.Date(2026, 11, 21, 20, 35, 0, 0, time.UTC)
quiet := queueOf{"main.137": `{"pipeline":"main.137","branch":"main","outcome":"running","source":137,"started":"2026-11-21T20:15:00Z",` +
`"movers":[{"module":"app"}],"steps":[{"module":"app","step":"build","outcome":"passed","status":"required","started":"2026-11-21T20:15:52Z","ended":"2026-11-21T20:17:55Z"}]}`}
var journal []string
health := HealthTool(quiet, none{}, func() time.Time { return now }, func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) })
for range 2 {
out, err := health.Run(nil)
if got, _ := json.Marshal(out); err != nil || !strings.Contains(string(got), "main.137 (nothing open and nothing placed for 17m5s)") {
t.Errorf("runner_health = %s (%v), want main.137 named stalled", got, err)
}
}
if len(journal) != 1 || !strings.Contains(journal[0], "main.137") {
t.Errorf("the journal said %q, want main.137 said once", journal)
}
}
// queueOf is a state queue in memory, its keys in no order.
@@ -7,6 +7,7 @@ import (
"strings"
"time"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/process"
)
@@ -19,13 +20,78 @@ const StateConsumers = process.ConsumersState
// behind (§76).
const BehindAfter = 5 * time.Minute
// Health is the runner's health as its tool runner_health answers it (§468, §76, §495; A-164): healthy when its state
// pipelines answers, as show reads it, and each durable consumer's oldest unacknowledged message waits less than
// BehindAfter; otherwise unhealthy naming each consumer behind and how long.
func Health(pipelines, consumers Kept, now time.Time) health.Answer {
if _, err := pipelines.Keys(); err != nil {
return health.Unhealthy(fmt.Sprintf("The runner does not answer show: its state pipelines cannot be listed (%v).", err))
// StalledAfter is how long a running pipeline may have nothing open and nothing newly placed before the runner's health
// names it stalled (H-A-311 (3)).
const StalledAfter = 5 * time.Minute
// Stalled is the key of each running pipeline, as show reads it, that has had nothing open (no step asked and not ended)
// and nothing newly placed, started or ended for StalledAfter at now (H-A-311 (3)), each with how long, in key order.
func Stalled(pipelines Kept, now time.Time) ([]string, error) {
all, err := Reads{Pipelines: pipelines}.pipelines()
if err != nil {
return nil, err
}
var out []string
for _, p := range all {
if p.Outcome != seat.PipelineOutcomeRunning {
continue
}
last, busy := startedOf(p), false
for _, st := range p.Steps {
if open(st) {
busy = true
break
}
for _, at := range []*time.Time{st.Started, st.Ended} {
if at != nil && at.After(last) {
last = *at
}
}
}
if busy || last.IsZero() || now.Sub(last) < StalledAfter {
continue
}
out = append(out, fmt.Sprintf("%s (nothing open and nothing placed for %s)", p.Pipeline, now.Sub(last).Round(time.Second)))
}
return out, nil
}
// Health is the runner's health as its tool runner_health answers it (§468, §76, §495; A-164): healthy when its state
// pipelines answers, as show reads it, each durable consumer's oldest unacknowledged message waits less than
// BehindAfter, and no running pipeline is stalled (H-A-311 (3)); otherwise unhealthy naming each consumer behind and
// how long, and each stalled pipeline by its key.
func Health(pipelines, consumers Kept, now time.Time) health.Answer {
a, _ := HealthStalled(pipelines, consumers, now)
return a
}
// HealthStalled is Health with the keys of the running pipelines it names stalled, so the tools program says each in its
// journal once (H-A-313).
func HealthStalled(pipelines, consumers Kept, now time.Time) (health.Answer, []string) {
if _, err := pipelines.Keys(); err != nil {
return health.Unhealthy(fmt.Sprintf("The runner does not answer show: its state pipelines cannot be listed (%v).", err)), nil
}
stalled, err := Stalled(pipelines, now)
if err != nil {
return health.Unhealthy(fmt.Sprintf("The runner's running pipelines cannot be read as show reads them (%v).", err)), nil
}
var keys []string
for _, s := range stalled {
keys = append(keys, strings.SplitN(s, " ", 2)[0])
}
a := consumersHealth(consumers, now)
if len(stalled) == 0 {
return a, nil
}
why := fmt.Sprintf("The runner's running pipelines with nothing open and nothing newly placed for %s or more: %s.", StalledAfter, strings.Join(stalled, ", "))
if !a.Healthy {
why = a.Why + " " + why
}
return health.Unhealthy(why), keys
}
// consumersHealth is the part of Health its consumers' ages give.
func consumersHealth(consumers Kept, now time.Time) health.Answer {
keys, err := consumers.Keys()
if err != nil {
return health.Unhealthy(fmt.Sprintf("The ages of the runner's consumers cannot be read from its state %s (%v).", StateConsumers, err))
@@ -7,6 +7,7 @@ import (
"testing"
"time"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-sdk/go/process"
)
@@ -50,6 +51,11 @@ func TestHealth(t *testing.T) {
{"six minutes names the consumer", mapKept{}, age("n_runner", 6*time.Minute), false, "n_runner (its oldest unacknowledged message waits 6m0s)"},
{"show does not answer", failingKept{}, age("n_runner", 0), false, "does not answer show"},
{"no event heard yet", mapKept{}, mapKept{}, true, "has heard no event yet"},
// H-A-311 (3): a running pipeline with nothing open and nothing newly placed for five minutes is named by its key.
{"a pipeline quiet six minutes is named", piped(now, "main.137", -6*time.Minute, seat.StepResultOutcomePassed), age("n_runner", 0), false,
"main.137 (nothing open and nothing placed for 6m0s)"},
{"a pipeline quiet four minutes is not", piped(now, "main.137", -4*time.Minute, seat.StepResultOutcomePassed), age("n_runner", 0), true, ""},
{"a pipeline with a step open is not", piped(now, "main.137", -20*time.Minute, seat.StepResultOutcomeRunning), age("n_runner", 0), true, ""},
}
for _, r := range rows {
a := Health(r.pipelines, r.consumers, now)
@@ -58,3 +64,16 @@ func TestHealth(t *testing.T) {
}
}
}
// piped is the state pipelines holding one running pipeline key whose one step was placed and last moved at now+moved,
// with outcome.
func piped(now time.Time, key string, moved time.Duration, outcome string) mapKept {
at, started := now.Add(moved), now.Add(-time.Hour)
st := seat.StepResult{Module: "app", Step: seat.StepResultStepBuild, Outcome: outcome, Status: seat.StepResultStatusRequired, Started: &at}
if outcome != seat.StepResultOutcomeRunning {
st.Ended = &at
}
raw, _ := json.Marshal(seat.Pipeline{Pipeline: key, Branch: "main", Outcome: seat.PipelineOutcomeRunning, Started: &started,
Movers: []seat.Mover{{Module: "app"}}, Steps: []seat.StepResult{st}})
return mapKept{key: raw}
}