From f22945c26ae6583617a544dc3afe166063fd4b4f Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 20:44:43 +0100 Subject: [PATCH] H-A-311 (3): runner_health names a running pipeline with nothing open and nothing placed for five minutes main.137 stood seventeen minutes with nothing open while runner_health said healthy. The tools program says each such pipeline in its journal once per key (H-A-313). Builds: design 47 H H-A-311 (3), H-A-313 --- .../cmd/mesh-delivery-pipeline-tools/main.go | 30 +++++-- .../mesh-delivery-pipeline-tools/main_test.go | 19 ++++- .../internal/pipeline/consumers.go | 78 +++++++++++++++++-- .../internal/pipeline/consumers_test.go | 19 +++++ 4 files changed, 133 insertions(+), 13 deletions(-) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go index 41964399e..facab7727 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go @@ -7,6 +7,7 @@ import ( "encoding/json" "fmt" "os" + "sync" "time" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" @@ -98,13 +99,29 @@ func FirstViewTools(module string, st stdio.KeptState, acts views.HandActs, now } // HealthTool is the runner's health check, run outside its process (design 47 G §76, §468, §495; A-164): it reads the -// runner's state pipelines as show does and the ages its process writes in the state consumers, and asks no seat verb. -func HealthTool(pipelines, consumers pipeline.Kept, now func() time.Time) stdio.Tool { +// runner's state pipelines as show does and the ages its process writes in the state consumers, and asks no seat verb. A +// running pipeline it first names stalled is said in this program's journal through logf, once per key for the +// program's life (H-A-311 (3), H-A-313). +func HealthTool(pipelines, consumers pipeline.Kept, now func() time.Time, logf func(string, ...any)) stdio.Tool { + var mu sync.Mutex + said := map[string]bool{} return stdio.Tool{Name: Prefix + "_health", - Description: "The pipeline runner's health: healthy when it answers show and each of its durable consumers' oldest " + - "unacknowledged message waits less than five minutes; otherwise it names each consumer behind.", + Description: "The pipeline runner's health: healthy when it answers show, each of its durable consumers' oldest " + + "unacknowledged message waits less than five minutes and no running pipeline has had nothing open and nothing " + + "newly placed for five minutes; otherwise it names each consumer behind and each such pipeline by its key.", Input: map[string]any{"type": "object", "properties": map[string]any{}}, - Run: func(map[string]any) (any, error) { return pipeline.Health(pipelines, consumers, now()), nil }} + Run: func(map[string]any) (any, error) { + a, stalled := pipeline.HealthStalled(pipelines, consumers, now()) + mu.Lock() + defer mu.Unlock() + for _, key := range stalled { + if !said[key] && logf != nil { + said[key] = true + logf("the running pipeline %s has had nothing open and nothing newly placed for %s or more: runner_health names it unhealthy (H-A-311)", key, pipeline.StalledAfter) + } + } + return a, nil + }} } // StoresFill is each of the runner's stores' fill through the bundle's calls (design 47 G §366, §623, A-214): the states @@ -154,7 +171,8 @@ func main() { tools = append(tools, CancelTool(r.List, stdio.Ask), RerunTool(stdio.Ask)) tools = append(tools, VerbTools(stdio.Ask)...) tools = append(tools, TestRunsTools(stdio.Ask)...) - tools = append(tools, HealthTool(r.Pipelines, kept{stdio.State(pipeline.StateConsumers)}, time.Now)) + tools = append(tools, HealthTool(r.Pipelines, kept{stdio.State(pipeline.StateConsumers)}, time.Now, + func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] "+f+"\n", a...) })) tools = append(tools, WaitingTool(kept{stdio.State(seat.StateQueue)})) if err := stdio.Serve("", tools); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] %v\n", err) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go index 95bb4bda1..1b2873b3a 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go @@ -3,6 +3,7 @@ package main import ( "encoding/json" "errors" + "fmt" "strings" "testing" @@ -37,7 +38,7 @@ func TestTools(t *testing.T) { names = append(names, tl.Name) } // RN25 (§468, §76; A-164): the health check is the bundle's runner_health. - names = append(names, HealthTool(none{}, none{}, nil).Name, WaitingTool(none{}).Name) + names = append(names, HealthTool(none{}, none{}, nil, nil).Name, WaitingTool(none{}).Name) // §606, §549: the unlogged entries are listed and appended by hand. for _, tl := range UnloggedTools(stdio.State("pipelines"), stdio.Log("step-log"), nil, nil) { names = append(names, tl.Name) @@ -53,6 +54,22 @@ func TestTools(t *testing.T) { if got, _ := json.Marshal(out); err != nil || string(got) != `[{"sequence":1,"pipeline":"main.5"},{"sequence":3,"pipeline":"main.7"}]` { t.Errorf("runner_waiting = %s (%v)", got, err) } + // H-A-311 (3), H-A-313: runner_health names a running pipeline quiet five minutes, and this program's journal says it + // once per key, however often the health is asked. + now := time.Date(2026, 11, 21, 20, 35, 0, 0, time.UTC) + quiet := queueOf{"main.137": `{"pipeline":"main.137","branch":"main","outcome":"running","source":137,"started":"2026-11-21T20:15:00Z",` + + `"movers":[{"module":"app"}],"steps":[{"module":"app","step":"build","outcome":"passed","status":"required","started":"2026-11-21T20:15:52Z","ended":"2026-11-21T20:17:55Z"}]}`} + var journal []string + health := HealthTool(quiet, none{}, func() time.Time { return now }, func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) }) + for range 2 { + out, err := health.Run(nil) + if got, _ := json.Marshal(out); err != nil || !strings.Contains(string(got), "main.137 (nothing open and nothing placed for 17m5s)") { + t.Errorf("runner_health = %s (%v), want main.137 named stalled", got, err) + } + } + if len(journal) != 1 || !strings.Contains(journal[0], "main.137") { + t.Errorf("the journal said %q, want main.137 said once", journal) + } } // queueOf is a state queue in memory, its keys in no order. diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/consumers.go b/modules/mesh-delivery-pipeline/internal/pipeline/consumers.go index d7a85fab6..2a0cddf07 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/consumers.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/consumers.go @@ -7,6 +7,7 @@ import ( "strings" "time" + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" "git.novox.be/novox/mesh-sdk/go/health" "git.novox.be/novox/mesh-sdk/go/process" ) @@ -19,13 +20,78 @@ const StateConsumers = process.ConsumersState // behind (§76). const BehindAfter = 5 * time.Minute -// Health is the runner's health as its tool runner_health answers it (§468, §76, §495; A-164): healthy when its state -// pipelines answers, as show reads it, and each durable consumer's oldest unacknowledged message waits less than -// BehindAfter; otherwise unhealthy naming each consumer behind and how long. -func Health(pipelines, consumers Kept, now time.Time) health.Answer { - if _, err := pipelines.Keys(); err != nil { - return health.Unhealthy(fmt.Sprintf("The runner does not answer show: its state pipelines cannot be listed (%v).", err)) +// StalledAfter is how long a running pipeline may have nothing open and nothing newly placed before the runner's health +// names it stalled (H-A-311 (3)). +const StalledAfter = 5 * time.Minute + +// Stalled is the key of each running pipeline, as show reads it, that has had nothing open (no step asked and not ended) +// and nothing newly placed, started or ended for StalledAfter at now (H-A-311 (3)), each with how long, in key order. +func Stalled(pipelines Kept, now time.Time) ([]string, error) { + all, err := Reads{Pipelines: pipelines}.pipelines() + if err != nil { + return nil, err } + var out []string + for _, p := range all { + if p.Outcome != seat.PipelineOutcomeRunning { + continue + } + last, busy := startedOf(p), false + for _, st := range p.Steps { + if open(st) { + busy = true + break + } + for _, at := range []*time.Time{st.Started, st.Ended} { + if at != nil && at.After(last) { + last = *at + } + } + } + if busy || last.IsZero() || now.Sub(last) < StalledAfter { + continue + } + out = append(out, fmt.Sprintf("%s (nothing open and nothing placed for %s)", p.Pipeline, now.Sub(last).Round(time.Second))) + } + return out, nil +} + +// Health is the runner's health as its tool runner_health answers it (§468, §76, §495; A-164): healthy when its state +// pipelines answers, as show reads it, each durable consumer's oldest unacknowledged message waits less than +// BehindAfter, and no running pipeline is stalled (H-A-311 (3)); otherwise unhealthy naming each consumer behind and +// how long, and each stalled pipeline by its key. +func Health(pipelines, consumers Kept, now time.Time) health.Answer { + a, _ := HealthStalled(pipelines, consumers, now) + return a +} + +// HealthStalled is Health with the keys of the running pipelines it names stalled, so the tools program says each in its +// journal once (H-A-313). +func HealthStalled(pipelines, consumers Kept, now time.Time) (health.Answer, []string) { + if _, err := pipelines.Keys(); err != nil { + return health.Unhealthy(fmt.Sprintf("The runner does not answer show: its state pipelines cannot be listed (%v).", err)), nil + } + stalled, err := Stalled(pipelines, now) + if err != nil { + return health.Unhealthy(fmt.Sprintf("The runner's running pipelines cannot be read as show reads them (%v).", err)), nil + } + var keys []string + for _, s := range stalled { + keys = append(keys, strings.SplitN(s, " ", 2)[0]) + } + a := consumersHealth(consumers, now) + if len(stalled) == 0 { + return a, nil + } + why := fmt.Sprintf("The runner's running pipelines with nothing open and nothing newly placed for %s or more: %s.", StalledAfter, strings.Join(stalled, ", ")) + if !a.Healthy { + why = a.Why + " " + why + } + return health.Unhealthy(why), keys +} + +// consumersHealth is the part of Health its consumers' ages give. +func consumersHealth(consumers Kept, now time.Time) health.Answer { keys, err := consumers.Keys() if err != nil { return health.Unhealthy(fmt.Sprintf("The ages of the runner's consumers cannot be read from its state %s (%v).", StateConsumers, err)) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/consumers_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/consumers_test.go index 8decf31c0..836aa602d 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/consumers_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/consumers_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" "git.novox.be/novox/mesh-sdk/go/process" ) @@ -50,6 +51,11 @@ func TestHealth(t *testing.T) { {"six minutes names the consumer", mapKept{}, age("n_runner", 6*time.Minute), false, "n_runner (its oldest unacknowledged message waits 6m0s)"}, {"show does not answer", failingKept{}, age("n_runner", 0), false, "does not answer show"}, {"no event heard yet", mapKept{}, mapKept{}, true, "has heard no event yet"}, + // H-A-311 (3): a running pipeline with nothing open and nothing newly placed for five minutes is named by its key. + {"a pipeline quiet six minutes is named", piped(now, "main.137", -6*time.Minute, seat.StepResultOutcomePassed), age("n_runner", 0), false, + "main.137 (nothing open and nothing placed for 6m0s)"}, + {"a pipeline quiet four minutes is not", piped(now, "main.137", -4*time.Minute, seat.StepResultOutcomePassed), age("n_runner", 0), true, ""}, + {"a pipeline with a step open is not", piped(now, "main.137", -20*time.Minute, seat.StepResultOutcomeRunning), age("n_runner", 0), true, ""}, } for _, r := range rows { a := Health(r.pipelines, r.consumers, now) @@ -58,3 +64,16 @@ func TestHealth(t *testing.T) { } } } + +// piped is the state pipelines holding one running pipeline key whose one step was placed and last moved at now+moved, +// with outcome. +func piped(now time.Time, key string, moved time.Duration, outcome string) mapKept { + at, started := now.Add(moved), now.Add(-time.Hour) + st := seat.StepResult{Module: "app", Step: seat.StepResultStepBuild, Outcome: outcome, Status: seat.StepResultStatusRequired, Started: &at} + if outcome != seat.StepResultOutcomeRunning { + st.Ended = &at + } + raw, _ := json.Marshal(seat.Pipeline{Pipeline: key, Branch: "main", Outcome: seat.PipelineOutcomeRunning, Started: &started, + Movers: []seat.Mover{{Module: "app"}}, Steps: []seat.StepResult{st}}) + return mapKept{key: raw} +}