Files
mesh-delivery/modules/mesh-delivery-watchman/internal/watch/steps.go
T
2026-11-17 13:03:57 +01:00

218 lines
7.5 KiB
Go

package watch
import (
"encoding/json"
"fmt"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
)
// The watchman times every step that reads a run (design 47 G §731, §732, A-468j, A-469, A-469o, A-469p): each step started and
// not ended is kept in memory under `<pipeline>/<step>/<artifact>` (the artifact empty for a module's step), its clock begun at its
// `step-started` and begun again at each `mesh-wants` `step-progress` whose ask is the step's own ask or one its `shared` names, at
// each build seat `step-progress` of such an ask, and at the `step-ended` of such an ask. A clock past the five-minute progress
// bound raises `pipeline-stands`, cleared at the step's own `step-ended` or its next sign of life. A step waiting on a person or
// queued for a build node (§86) is owed no progress. At its start the watchman seeds its open steps from the runner's `stands`.
// stepRow is the signals table's row of a step standing still.
const stepRow = "step-stands"
// openStep is a step started and not ended.
type openStep struct {
pipeline, step, artifact string
// asks are the step's own ask and the asks its shared runs name; ask the one last heard of.
asks map[string]bool
ask string
// at is when the clock began; owed says whether the step is owed progress.
at time.Time
owed bool
}
// stepKey is the key an open step is kept under.
func stepKey(pipeline, step, artifact string) string { return pipeline + "/" + step + "/" + artifact }
// StepFindingID is the finding's id of an open step.
func StepFindingID(key string) string { return "step-" + unsafeKey.ReplaceAllString(key, "_") }
// StepStarted hears the runner's `step-started`: the step is open and its clock begins at the event's time, unless it waits on a
// person or for a build node, which are owed no progress. One that is no running or waiting step is ended.
func (w *Watchman) StepStarted(body []byte) {
var e meshpipelinerunner.StepEvent
if err := json.Unmarshal(body, &e); err != nil || e.Pipeline == "" || e.Result.Step == "" {
w.say("a step-started does not read, left: %v", err)
return
}
key := stepKey(e.Pipeline, e.Result.Step, e.Result.Artifact)
w.mu.Lock()
defer w.mu.Unlock()
switch e.Result.Outcome {
case meshpipelinerunner.StepResultOutcomeRunning, "", meshpipelinerunner.StepResultOutcomeWaiting,
meshpipelinerunner.StepResultOutcomeWaitingForABuildNode:
default:
w.closeStep(key)
return
}
at := w.now()
switch {
case e.At != nil && !e.At.IsZero():
at = *e.At
case e.Result.Started != nil && !e.Result.Started.IsZero():
at = *e.Result.Started
}
owed := e.Result.Outcome == meshpipelinerunner.StepResultOutcomeRunning || e.Result.Outcome == ""
w.openStep(key, e.Pipeline, e.Result.Step, e.Result.Artifact, e.Result.Ask, sharedAsks(e.Result.Shared), at, owed)
}
// StepEnded hears the runner's `step-ended`: the step's clock ends and its finding is cleared.
func (w *Watchman) StepEnded(body []byte) {
var e meshpipelinerunner.StepEvent
if err := json.Unmarshal(body, &e); err != nil || e.Pipeline == "" || e.Result.Step == "" {
w.say("a step-ended does not read, left: %v", err)
return
}
w.mu.Lock()
defer w.mu.Unlock()
w.closeStep(stepKey(e.Pipeline, e.Result.Step, e.Result.Artifact))
}
// WantsStepProgress hears the seat mesh-wants' `step-progress` (the assigner's, of a composition step): a step whose ask it is, or
// a shared ask it names, is alive; the runs it names are kept as the step's.
func (w *Watchman) WantsStepProgress(body []byte) {
var e meshwants.StepProgressEvent
if err := json.Unmarshal(body, &e); err != nil || e.Ask == "" {
w.say("a step-progress of mesh-wants does not read, left: %v", err)
return
}
w.mu.Lock()
defer w.mu.Unlock()
var named []string
for _, s := range e.Shared {
if s.Ask != "" {
named = append(named, s.Ask)
}
}
for key, s := range w.steps {
if !s.asks[e.Ask] {
continue
}
for _, a := range named {
s.asks[a] = true
}
w.stepAlive(key, e.Ask, e.At)
}
}
// stepsHeardOf says the steps that name ask are alive: a build seat said its progress or its end (§732).
func (w *Watchman) stepsHeardOf(ask string, at time.Time) {
if ask == "" {
return
}
for key, s := range w.steps {
if s.asks[ask] {
w.stepAlive(key, ask, at)
}
}
}
// stepAlive begins an owed step's clock again at at, never earlier than it began, w.mu held.
func (w *Watchman) stepAlive(key, ask string, at time.Time) {
s := w.steps[key]
if at.IsZero() {
at = w.now()
}
s.ask = ask
if at.After(s.at) {
s.at = at
}
w.timeStep(key)
}
func sharedAsks(shared []meshpipelinerunner.Shared) []string {
var out []string
for _, s := range shared {
if s.Ask != "" {
out = append(out, s.Ask)
}
}
return out
}
// openStep keeps a step and times it, w.mu held. A step kept before keeps the asks it knew.
func (w *Watchman) openStep(key, pipeline, step, artifact, ask string, shared []string, at time.Time, owed bool) {
s := w.steps[key]
if s == nil {
s = &openStep{pipeline: pipeline, step: step, artifact: artifact, asks: map[string]bool{}}
w.steps[key] = s
}
s.at, s.owed = at, owed
if ask != "" {
s.asks[ask], s.ask = true, ask
}
for _, a := range shared {
s.asks[a] = true
}
w.timeStep(key)
}
// closeStep ends a step, w.mu held.
func (w *Watchman) closeStep(key string) {
if _, ok := w.steps[key]; !ok {
return
}
delete(w.steps, key)
id := StepFindingID(key)
w.unsignal(stepRow, id)
w.unfind(FindingKey(ScopeDelivery, id, KindPipelineStands))
}
// timeStep times an open step, w.mu held: the progress bound from its clock, the finding raised at once when that has passed.
func (w *Watchman) timeStep(key string) {
s := w.steps[key]
id := StepFindingID(key)
w.unsignal(stepRow, id)
w.unfind(FindingKey(ScopeDelivery, id, KindPipelineStands))
if !s.owed {
return
}
name := s.pipeline + " " + s.step
if s.artifact != "" {
name += " " + s.artifact
}
last := ""
if s.ask != "" {
last = ", last heard of its ask " + s.ask
}
f := Finding{Scope: ScopeDelivery, ID: id, Kind: KindPipelineStands, Severity: Warning, Source: stepRow, Since: s.at,
Summary: fmt.Sprintf("the step %s has said nothing since %s%s", name, stamp(s.at), last),
Said: "no progress in " + w.progressBound.String(),
Headline: "A pipeline step has stopped saying it lives",
Explanation: "The step " + name + " started and has not ended, and neither it nor the run it reads has said its progress for " +
w.progressBound.String() + " or more" + last + ".",
Needs: "the part that runs it to say its progress, or the step ended (the runner's verbs mesh-pipeline-runner.stands and mesh-pipeline-runner.cancel)"}
if left := w.progressBound - w.now().Sub(s.at); left > 0 {
w.signal(stepRow, id, left, signalDue{Do: dueFind, Finding: f})
return
}
w.find(f)
}
// SeedSteps seeds the open steps from the runner's `stands` at the watchman's start (§731, A-468j): each step not already kept
// begins its clock now, and a step that waits on a person or for a build node is owed none.
func (w *Watchman) SeedSteps(stands []meshpipelinerunner.Wait) {
w.mu.Lock()
defer w.mu.Unlock()
for _, s := range stands {
if s.Pipeline == "" || s.Step == "" {
continue
}
key := stepKey(s.Pipeline, s.Step, s.Artifact)
if _, ok := w.steps[key]; ok {
continue
}
owed := s.On != meshpipelinerunner.WaitOnPerson && s.On != meshpipelinerunner.WaitOnBuildNode && s.Who == ""
w.openStep(key, s.Pipeline, s.Step, s.Artifact, s.Ask, sharedAsks(s.Shared), w.now(), owed)
}
}