R9: the stalled bound is the setting stalled-bound; the health names a step whose ask is untaken past it

Builds: design 47 H2 §790, §778 (the health line), §781 (read through mesh-sdk's settings reader)
This commit is contained in:
2026-11-22 01:23:58 +01:00
parent 43a88ef96c
commit d13efa8abe
8 changed files with 118 additions and 27 deletions
@@ -103,17 +103,18 @@ func FirstViewTools(module string, st stdio.KeptState, acts views.HandActs, now
// runner's state pipelines as show does and the ages its process writes in the state consumers, and asks no seat verb. A
// running pipeline it first names stalled is said in this program's journal through logf, once per key for the
// program's life (H-A-311 (3), H-A-313).
func HealthTool(pipelines, consumers, firstViews pipeline.Kept, now func() time.Time, logf func(string, ...any)) stdio.Tool {
func HealthTool(pipelines, consumers, firstViews pipeline.Kept, bound time.Duration, now func() time.Time, logf func(string, ...any)) stdio.Tool {
var mu sync.Mutex
said := map[string]bool{}
return stdio.Tool{Name: Prefix + "_health",
Description: "The pipeline runner's health: healthy when it answers show, each of its durable consumers' oldest " +
"unacknowledged message waits less than five minutes and no running pipeline has had nothing open and nothing " +
"newly placed for five minutes; otherwise it names each consumer behind and each such pipeline by its key. It " +
"newly placed, nor a step's ask stayed untaken, for the setting stalled-bound (five minutes by default); otherwise it " +
"names each consumer behind, each such pipeline by its key and each such step. It " +
"answers registrations-read: whether the runner's registrations view's first read is in, and since when.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) {
a, stalled := pipeline.HealthStalled(pipelines, consumers, now())
a, stalled := pipeline.HealthStalled(pipelines, consumers, now(), bound)
// design 47 H2 §788: whether the registrations are read, from the runner's own first-views state.
read, err := pipeline.RegistrationsReadOf(firstViews)
if err != nil {
@@ -124,7 +125,7 @@ func HealthTool(pipelines, consumers, firstViews pipeline.Kept, now func() time.
for _, key := range stalled {
if !said[key] && logf != nil {
said[key] = true
logf("the running pipeline %s has had nothing open and nothing newly placed for %s or more: runner_health names it unhealthy (H-A-311)", key, pipeline.StalledAfter)
logf("%s has stood still for %s or more (nothing open and nothing newly placed, or its ask untaken): runner_health names it unhealthy (H-A-311, design 47 H2 §790)", key, bound)
}
}
return HealthAnswer{Answer: a, RegistrationsRead: read}, nil
@@ -174,6 +175,12 @@ func StoresFill(state func(string) (*stdio.StateInfo, error), objects func(strin
}
func main() {
// design 47 H2 §790, §781: the stalled bound is the runner's setting, read here as its process reads it.
stalledBound, err := stdio.SettingDuration(pipeline.StalledBoundSetting, pipeline.DefaultStalledBound)
if err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] %v\n", err)
os.Exit(1)
}
fill := StoresFill(func(n string) (*stdio.StateInfo, error) { return stdio.State(n).Info() },
func(n string) (buscalls.Fill, error) { return busstate.ObjectsOf(n).Fill() },
func(n string) (*stdio.LogInfo, error) { return stdio.Log(n).Info() }, time.Now)
@@ -185,7 +192,7 @@ func main() {
tools = append(tools, CancelTool(r.List, stdio.Ask), RerunTool(stdio.Ask))
tools = append(tools, VerbTools(stdio.Ask)...)
tools = append(tools, TestRunsTools(stdio.Ask)...)
tools = append(tools, HealthTool(r.Pipelines, kept{stdio.State(pipeline.StateConsumers)}, kept{stdio.State(views.StatusState)}, time.Now,
tools = append(tools, HealthTool(r.Pipelines, kept{stdio.State(pipeline.StateConsumers)}, kept{stdio.State(views.StatusState)}, stalledBound, time.Now,
func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] "+f+"\n", a...) }))
tools = append(tools, WaitingTool(kept{stdio.State(seat.StateQueue)}))
tools = append(tools, UnfinishedTools(keptDelete{kept{stdio.State(pipeline.StateUnfinished)}}, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
@@ -42,7 +42,7 @@ func TestTools(t *testing.T) {
names = append(names, tl.Name)
}
// RN25 (§468, §76; A-164): the health check is the bundle's runner_health.
names = append(names, HealthTool(none{}, none{}, none{}, nil, nil).Name, WaitingTool(none{}).Name)
names = append(names, HealthTool(none{}, none{}, none{}, pipeline.DefaultStalledBound, nil, nil).Name, WaitingTool(none{}).Name)
// §606, §549: the unlogged entries are listed and appended by hand.
for _, tl := range UnloggedTools(stdio.State("pipelines"), stdio.Log("step-log"), nil, nil) {
names = append(names, tl.Name)
@@ -71,7 +71,7 @@ func TestTools(t *testing.T) {
`"movers":[{"module":"app"}],"steps":[{"module":"app","step":"build","outcome":"passed","status":"required","started":"2026-11-21T20:15:52Z","ended":"2026-11-21T20:17:55Z"}]}`}
var journal []string
firstViews := queueOf{}
health := HealthTool(quiet, none{}, firstViews, func() time.Time { return now }, func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) })
health := HealthTool(quiet, none{}, firstViews, pipeline.DefaultStalledBound, func() time.Time { return now }, func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) })
for range 2 {
out, err := health.Run(nil)
if got, _ := json.Marshal(out); err != nil || !strings.Contains(string(got), "main.137 (nothing open and nothing placed for 17m5s)") {
@@ -20,6 +20,7 @@ import (
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatlist"
@@ -166,6 +167,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err != nil {
return err
}
// design 47 H2 §790, §781: the stalled bound, read as the tools program reads it; a value that is no duration stops the start.
stalledBound, err := stdio.SettingDuration(pipeline.StalledBoundSetting, pipeline.DefaultStalledBound)
if err != nil {
return err
}
logger.Printf("health: a running pipeline with nothing open and nothing newly placed, or a step whose ask is untaken, is named after %s (%s)", stalledBound, pipeline.StalledBoundSetting)
// Its asking half (§22, §24): a pipeline's steps are placed from the module's registration and each build seat is asked
// for its step.
starter := &pipeline.Starter{Pipelines: store, Logf: logger.Printf, Say: say,
@@ -22,6 +22,7 @@ import (
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatlist"
@@ -50,6 +51,8 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
reg := catalog.Registration{Module: "app", Branches: []string{"main"}, Origin: "o/app", Policy: catalog.Policy{Steps: catalog.StepStatuses{Review: "optional", Validate: "bypass", Composition: "bypass", Approval: "bypass"}}}
// design 47 H2 §790: the stalled bound is the setting stalled-bound, here one minute, read as the node-engine composes it.
t.Setenv(stdio.SettingWord(pipeline.StalledBoundSetting), "1m")
// The branch main has no revision yet at the runner's start (none-yet), then revision 1 holding app (A-336).
var revised atomic.Bool
w := startWhole(t, ctx, []catalog.Registration{reg}, func([]byte) catalog.RevisionAnswer {
@@ -77,6 +80,15 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes
if len(p.Steps) != 0 {
t.Fatalf("a take-in before its branch's half placed %d steps, want none (A-336)", len(p.Steps))
}
// design 47 H2 §790: at one minute the health names the pipeline with nothing placed, a minute after its start.
bound, err := stdio.SettingDuration(pipeline.StalledBoundSetting, pipeline.DefaultStalledBound)
if err != nil || bound != time.Minute || !strings.Contains(logs.String(), "is named after 1m0s (stalled-bound)") {
t.Fatalf("the stalled bound %s (%v); the process says it: %t", bound, err, strings.Contains(logs.String(), "is named after 1m0s"))
}
noConsumers := kvKept{ctx, mustKV(t, ctx, js, module+"_consumers")}
if a := pipeline.Health(kvKept{ctx, pipelines}, noConsumers, p.Started.Add(61*time.Second), bound); a.Healthy || !strings.Contains(a.Why, key+" (nothing open") {
t.Fatalf("a minute after its start, with nothing placed, the health says %+v", a)
}
// The revision said is main's first half: held, it places the pipeline's steps from the registration (A-327).
revised.Store(true)
rev, _ := json.Marshal(catalog.CatalogRevisionEvent{V: 1, Revision: 1, Kind: "trunk", Branch: "main", Source: 3})
@@ -88,6 +100,10 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes
if build.Outcome != seat.StepResultOutcomeRunning || build.Ask == "" {
t.Fatalf("the build step = %+v", build)
}
// Once a step is placed in it, the health no longer names it (§790).
if a := pipeline.Health(kvKept{ctx, pipelines}, noConsumers, p.Started.Add(61*time.Second), bound); strings.Contains(a.Why, key+" (nothing open") {
t.Fatalf("with its build placed, the health still names %s: %+v", key, a)
}
// The build is asked of the seat mesh-builds: its ask is on its work queue.
waitFor(t, ctx, logs, func() bool { info, err := asks.Info(ctx); return err == nil && info.State.Msgs >= 1 })
@@ -167,7 +183,7 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes
e, err := consumers.Get(ctx, "n1_"+module)
return err == nil && json.Unmarshal(e.Value(), &facts) == nil && facts.OldestUnacknowledged.IsZero() && facts.Acknowledged > 0
})
if a := pipeline.Health(kvKept{ctx, pipelines}, kvKept{ctx, consumers}, time.Now()); !a.Healthy {
if a := pipeline.Health(kvKept{ctx, pipelines}, kvKept{ctx, consumers}, time.Now(), bound); !a.Healthy {
t.Errorf("runner_health over the bus: %+v", a)
}
}
@@ -293,6 +309,16 @@ func startWhole(t *testing.T, ctx context.Context, regs []catalog.Registration,
return whole{nc: nc, js: js, asks: asks, pipelines: pipelines, logs: logs}
}
// mustKV is the bucket name.
func mustKV(t *testing.T, ctx context.Context, js jetstream.JetStream, name string) jetstream.KeyValue {
t.Helper()
kv, err := js.KeyValue(ctx, name)
if err != nil {
t.Fatal(err)
}
return kv
}
// kvKept reads a bucket as the tools bundle's states do.
type kvKept struct {
ctx context.Context
@@ -22,32 +22,39 @@ const StateConsumers = process.ConsumersState
// behind (§76).
const BehindAfter = 5 * time.Minute
// StalledAfter is how long a running pipeline may have nothing open and nothing newly placed before the runner's health
// names it stalled (H-A-311 (3)): the mesh's own rule for nothing moving (nothing-moved-in-five-minutes), not a setting
// (H-A-322).
const StalledAfter = 5 * time.Minute
// DefaultStalledBound is the setting stalled-bound's default (design 47 H2 §790): how long a running pipeline may have
// nothing open and nothing newly placed, or a step's ask stay untaken (§778), before the runner's health names it; the
// mesh's own rule for nothing moving (nothing-moved-in-five-minutes).
const DefaultStalledBound = 5 * time.Minute
// StalledBoundSetting is the runner's setting of that bound, read by its process and its tools program (§790, §781).
const StalledBoundSetting = "stalled-bound"
// Stalled is the key of each running pipeline, as show reads it, that has had nothing open (no step asked and not ended,
// no required step waiting but on a package's publish) and nothing newly placed, started or ended for StalledAfter at now (H-A-311 (3)), each with
// how long, in key order.
func Stalled(pipelines Kept, now time.Time) ([]string, error) {
// no required step waiting but on a package's publish) and nothing newly placed, started or ended for bound at now
// (H-A-311 (3), design 47 H2 §790), each with how long, in key order; and each step of a running pipeline whose ask is
// still untaken bound after it was put (§778), each with its seat and how long.
func Stalled(pipelines Kept, now time.Time, bound time.Duration) ([]string, error) {
all, err := Reads{Pipelines: pipelines}.pipelines()
if err != nil {
return nil, err
}
var out []string
var out, untaken []string
for _, p := range all {
if p.Outcome != seat.PipelineOutcomeRunning {
continue
}
last, busy := startedOf(p), false
for _, st := range p.Steps {
if st.Ask != "" && st.Ended == nil && st.TakenAt == nil && st.AskedAt != nil && open(st) && now.Sub(*st.AskedAt) >= bound {
untaken = append(untaken, fmt.Sprintf("%s %s %s (its ask on %s untaken for %s)", p.Pipeline, st.Module, st.Step,
st.Seat, now.Sub(*st.AskedAt).Round(time.Second)))
}
// A required step waiting (an approval, a composition on its validate) is open, as the end's judgement holds it;
// an optional one waiting holds nothing (A-104), and neither does one waiting on a package's publish, as a mover
// with no steps waiting on one does not (H-A-326 (2)).
if open(st) || (st.Outcome == seat.StepResultOutcomeWaiting && st.Status == seat.StepResultStatusRequired && !pinHeld(st)) {
busy = true
break
}
for _, at := range []*time.Time{st.Started, st.Ended} {
if at != nil && at.After(last) {
@@ -55,42 +62,42 @@ func Stalled(pipelines Kept, now time.Time) ([]string, error) {
}
}
}
if busy || last.IsZero() || now.Sub(last) < StalledAfter {
if busy || last.IsZero() || now.Sub(last) < bound {
continue
}
out = append(out, fmt.Sprintf("%s (nothing open and nothing placed for %s)", p.Pipeline, now.Sub(last).Round(time.Second)))
}
return out, nil
return append(out, untaken...), nil
}
// Health is the runner's health as its tool runner_health answers it (§468, §76, §495; A-164): healthy when its state
// pipelines answers, as show reads it, each durable consumer's oldest unacknowledged message waits less than
// BehindAfter, and no running pipeline is stalled (H-A-311 (3)); otherwise unhealthy naming each consumer behind and
// how long, and each stalled pipeline by its key.
func Health(pipelines, consumers Kept, now time.Time) health.Answer {
a, _ := HealthStalled(pipelines, consumers, now)
func Health(pipelines, consumers Kept, now time.Time, bound time.Duration) health.Answer {
a, _ := HealthStalled(pipelines, consumers, now, bound)
return a
}
// HealthStalled is Health with the keys of the running pipelines it names stalled, so the tools program says each in its
// journal once (H-A-313).
func HealthStalled(pipelines, consumers Kept, now time.Time) (health.Answer, []string) {
func HealthStalled(pipelines, consumers Kept, now time.Time, bound time.Duration) (health.Answer, []string) {
if _, err := pipelines.Keys(); err != nil {
return health.Unhealthy(fmt.Sprintf("The runner does not answer show: its state pipelines cannot be listed (%v).", err)), nil
}
stalled, err := Stalled(pipelines, now)
stalled, err := Stalled(pipelines, now, bound)
if err != nil {
return health.Unhealthy(fmt.Sprintf("The runner's running pipelines cannot be read as show reads them (%v).", err)), nil
}
var keys []string
for _, s := range stalled {
keys = append(keys, strings.SplitN(s, " ", 2)[0])
keys = append(keys, strings.SplitN(s, " (", 2)[0])
}
a := consumersHealth(consumers, now)
if len(stalled) == 0 {
return a, nil
}
why := fmt.Sprintf("The runner's running pipelines with nothing open and nothing newly placed for %s or more: %s.", StalledAfter, strings.Join(stalled, ", "))
why := fmt.Sprintf("The runner's running pipelines with nothing open and nothing newly placed, or a step whose ask is untaken, for %s or more: %s.", bound, strings.Join(stalled, ", "))
if !a.Healthy {
why = a.Why + " " + why
}
@@ -65,8 +65,37 @@ func TestHealth(t *testing.T) {
{"a pipeline waiting only on packages is named", heldOnPublish(piped(now, "main.137", -20*time.Minute, seat.StepResultOutcomeWaiting), "main.137"), age("n_runner", 0), false,
"main.137 (nothing open and nothing placed for 20m0s)"},
}
// design 47 H2 §778, §790: a step whose ask is untaken past the bound is named with its seat; the bound is the setting's.
untaken := func(asked time.Duration, taken bool) mapKept {
k := piped(now, "main.140", -time.Minute, seat.StepResultOutcomeWaitingForABuildNode)
var p seat.Pipeline
_ = json.Unmarshal(k["main.140"], &p)
at := now.Add(-asked)
p.Steps[0].Ask, p.Steps[0].Seat, p.Steps[0].AskedAt, p.Steps[0].Ended = "ask-1", "mesh-builds", &at, nil
if taken {
p.Steps[0].TakenAt = &at
}
raw, _ := json.Marshal(p)
k["main.140"] = raw
return k
}
rows = append(rows, []struct {
name string
pipelines Kept
consumers Kept
healthy bool
says string
}{
{"an ask untaken six minutes is named", untaken(6*time.Minute, false), age("n_runner", 0), false, "main.140 app build (its ask on mesh-builds untaken for 6m0s)"},
{"an ask untaken four minutes is not", untaken(4*time.Minute, false), age("n_runner", 0), true, ""},
{"a taken ask is not", untaken(20*time.Minute, true), age("n_runner", 0), true, ""},
}...)
if a := Health(piped(now, "main.137", -2*time.Minute, seat.StepResultOutcomePassed), age("n_runner", 0), now, time.Minute); a.Healthy ||
!strings.Contains(a.Why, "main.137 (nothing open and nothing placed for 2m0s)") {
t.Errorf("at a one-minute bound a pipeline quiet two minutes: %q", a.Why)
}
for _, r := range rows {
a := Health(r.pipelines, r.consumers, now)
a := Health(r.pipelines, r.consumers, now, DefaultStalledBound)
if a.Healthy != r.healthy || !strings.Contains(a.Why, r.says) {
t.Errorf("%s: healthy %v %q; want %v saying %q", r.name, a.Healthy, a.Why, r.healthy, r.says)
}
@@ -207,7 +207,15 @@ func runSettingsOf(raw []byte) bool {
}
want := map[string]string{"test-runs-cap-mib": "number", "test-runs-kept": "string", "test-step-bound": "string", "run-untaken-bound": "string", "ask-unanswered-bound": "string", "ask-silent-bound": "string", "widens-read-bound": "string", "revisions-kept": "number",
"pipelines-at-once": "number", "pipelines-waiting-cap": "number"}
if len(m.Config) != len(want) || m.Config["test-runs-cap-mib"]["maximum"] != float64(64) || m.Config["test-runs-cap-mib"]["default"] != float64(16) ||
// design 47 H2 §790, §781: stalled-bound is read through mesh-sdk's settings reader, by the process and the tools
// program alike, from the words the node-engine composes: no file of its own.
words := map[string]string{"stalled-bound": "string"}
for key, kind := range words {
if m.Config[key]["type"] != kind || m.Config[key]["default"] != "5m" || m.Config[key]["runtime"] != true {
return false
}
}
if len(m.Config) != len(want)+len(words) || m.Config["test-runs-cap-mib"]["maximum"] != float64(64) || m.Config["test-runs-cap-mib"]["default"] != float64(16) ||
m.Config["pipelines-at-once"]["minimum"] != float64(1) || m.Config["pipelines-at-once"]["maximum"] != float64(8) ||
m.Config["pipelines-at-once"]["default"] != float64(1) || m.Config["pipelines-waiting-cap"]["default"] != float64(64) {
return false
@@ -167,6 +167,13 @@
"trusted": false,
"default": "10m"
},
"stalled-bound": {
"type": "string",
"description": "how long a running pipeline may have nothing open and nothing newly placed, or a step's ask stay untaken, before the runner's health names it, a Go duration such as 5m (design 47 H2 sections 778, 790).",
"trusted": false,
"runtime": true,
"default": "5m"
},
"ask-silent-bound": {
"type": "string",
"description": "how long an ask the runner puts on a seat whose holder says nothing before the step's end (mesh-builds, mesh-wants) may go without that end before its step ends could not run, a Go duration such as 45m (design 47 H H-A-243).",