218 lines
7.5 KiB
Go
218 lines
7.5 KiB
Go
package watch
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
|
)
|
|
|
|
// The watchman times every step that reads a run (design 47 G §731, §732, A-468j, A-469, A-469o, A-469p): each step started and
|
|
// not ended is kept in memory under `<pipeline>/<step>/<artifact>` (the artifact empty for a module's step), its clock begun at its
|
|
// `step-started` and begun again at each `mesh-wants` `step-progress` whose ask is the step's own ask or one its `shared` names, at
|
|
// each build seat `step-progress` of such an ask, and at the `step-ended` of such an ask. A clock past the five-minute progress
|
|
// bound raises `pipeline-stands`, cleared at the step's own `step-ended` or its next sign of life. A step waiting on a person or
|
|
// queued for a build node (§86) is owed no progress. At its start the watchman seeds its open steps from the runner's `stands`.
|
|
|
|
// stepRow is the signals table's row of a step standing still.
|
|
const stepRow = "step-stands"
|
|
|
|
// openStep is a step started and not ended.
|
|
type openStep struct {
|
|
pipeline, step, artifact string
|
|
// asks are the step's own ask and the asks its shared runs name; ask the one last heard of.
|
|
asks map[string]bool
|
|
ask string
|
|
// at is when the clock began; owed says whether the step is owed progress.
|
|
at time.Time
|
|
owed bool
|
|
}
|
|
|
|
// stepKey is the key an open step is kept under.
|
|
func stepKey(pipeline, step, artifact string) string { return pipeline + "/" + step + "/" + artifact }
|
|
|
|
// StepFindingID is the finding's id of an open step.
|
|
func StepFindingID(key string) string { return "step-" + unsafeKey.ReplaceAllString(key, "_") }
|
|
|
|
// StepStarted hears the runner's `step-started`: the step is open and its clock begins at the event's time, unless it waits on a
|
|
// person or for a build node, which are owed no progress. One that is no running or waiting step is ended.
|
|
func (w *Watchman) StepStarted(body []byte) {
|
|
var e meshpipelinerunner.StepEvent
|
|
if err := json.Unmarshal(body, &e); err != nil || e.Pipeline == "" || e.Result.Step == "" {
|
|
w.say("a step-started does not read, left: %v", err)
|
|
return
|
|
}
|
|
key := stepKey(e.Pipeline, e.Result.Step, e.Result.Artifact)
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
switch e.Result.Outcome {
|
|
case meshpipelinerunner.StepResultOutcomeRunning, "", meshpipelinerunner.StepResultOutcomeWaiting,
|
|
meshpipelinerunner.StepResultOutcomeWaitingForABuildNode:
|
|
default:
|
|
w.closeStep(key)
|
|
return
|
|
}
|
|
at := w.now()
|
|
switch {
|
|
case e.At != nil && !e.At.IsZero():
|
|
at = *e.At
|
|
case e.Result.Started != nil && !e.Result.Started.IsZero():
|
|
at = *e.Result.Started
|
|
}
|
|
owed := e.Result.Outcome == meshpipelinerunner.StepResultOutcomeRunning || e.Result.Outcome == ""
|
|
w.openStep(key, e.Pipeline, e.Result.Step, e.Result.Artifact, e.Result.Ask, sharedAsks(e.Result.Shared), at, owed)
|
|
}
|
|
|
|
// StepEnded hears the runner's `step-ended`: the step's clock ends and its finding is cleared.
|
|
func (w *Watchman) StepEnded(body []byte) {
|
|
var e meshpipelinerunner.StepEvent
|
|
if err := json.Unmarshal(body, &e); err != nil || e.Pipeline == "" || e.Result.Step == "" {
|
|
w.say("a step-ended does not read, left: %v", err)
|
|
return
|
|
}
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
w.closeStep(stepKey(e.Pipeline, e.Result.Step, e.Result.Artifact))
|
|
}
|
|
|
|
// WantsStepProgress hears the seat mesh-wants' `step-progress` (the assigner's, of a composition step): a step whose ask it is, or
|
|
// a shared ask it names, is alive; the runs it names are kept as the step's.
|
|
func (w *Watchman) WantsStepProgress(body []byte) {
|
|
var e meshwants.StepProgressEvent
|
|
if err := json.Unmarshal(body, &e); err != nil || e.Ask == "" {
|
|
w.say("a step-progress of mesh-wants does not read, left: %v", err)
|
|
return
|
|
}
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
var named []string
|
|
for _, s := range e.Shared {
|
|
if s.Ask != "" {
|
|
named = append(named, s.Ask)
|
|
}
|
|
}
|
|
for key, s := range w.steps {
|
|
if !s.asks[e.Ask] {
|
|
continue
|
|
}
|
|
for _, a := range named {
|
|
s.asks[a] = true
|
|
}
|
|
w.stepAlive(key, e.Ask, e.At)
|
|
}
|
|
}
|
|
|
|
// stepsHeardOf says the steps that name ask are alive: a build seat said its progress or its end (§732).
|
|
func (w *Watchman) stepsHeardOf(ask string, at time.Time) {
|
|
if ask == "" {
|
|
return
|
|
}
|
|
for key, s := range w.steps {
|
|
if s.asks[ask] {
|
|
w.stepAlive(key, ask, at)
|
|
}
|
|
}
|
|
}
|
|
|
|
// stepAlive begins an owed step's clock again at at, never earlier than it began, w.mu held.
|
|
func (w *Watchman) stepAlive(key, ask string, at time.Time) {
|
|
s := w.steps[key]
|
|
if at.IsZero() {
|
|
at = w.now()
|
|
}
|
|
s.ask = ask
|
|
if at.After(s.at) {
|
|
s.at = at
|
|
}
|
|
w.timeStep(key)
|
|
}
|
|
|
|
func sharedAsks(shared []meshpipelinerunner.Shared) []string {
|
|
var out []string
|
|
for _, s := range shared {
|
|
if s.Ask != "" {
|
|
out = append(out, s.Ask)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// openStep keeps a step and times it, w.mu held. A step kept before keeps the asks it knew.
|
|
func (w *Watchman) openStep(key, pipeline, step, artifact, ask string, shared []string, at time.Time, owed bool) {
|
|
s := w.steps[key]
|
|
if s == nil {
|
|
s = &openStep{pipeline: pipeline, step: step, artifact: artifact, asks: map[string]bool{}}
|
|
w.steps[key] = s
|
|
}
|
|
s.at, s.owed = at, owed
|
|
if ask != "" {
|
|
s.asks[ask], s.ask = true, ask
|
|
}
|
|
for _, a := range shared {
|
|
s.asks[a] = true
|
|
}
|
|
w.timeStep(key)
|
|
}
|
|
|
|
// closeStep ends a step, w.mu held.
|
|
func (w *Watchman) closeStep(key string) {
|
|
if _, ok := w.steps[key]; !ok {
|
|
return
|
|
}
|
|
delete(w.steps, key)
|
|
id := StepFindingID(key)
|
|
w.unsignal(stepRow, id)
|
|
w.unfind(FindingKey(ScopeDelivery, id, KindPipelineStands))
|
|
}
|
|
|
|
// timeStep times an open step, w.mu held: the progress bound from its clock, the finding raised at once when that has passed.
|
|
func (w *Watchman) timeStep(key string) {
|
|
s := w.steps[key]
|
|
id := StepFindingID(key)
|
|
w.unsignal(stepRow, id)
|
|
w.unfind(FindingKey(ScopeDelivery, id, KindPipelineStands))
|
|
if !s.owed {
|
|
return
|
|
}
|
|
name := s.pipeline + " " + s.step
|
|
if s.artifact != "" {
|
|
name += " " + s.artifact
|
|
}
|
|
last := ""
|
|
if s.ask != "" {
|
|
last = ", last heard of its ask " + s.ask
|
|
}
|
|
f := Finding{Scope: ScopeDelivery, ID: id, Kind: KindPipelineStands, Severity: Warning, Source: stepRow, Since: s.at,
|
|
Summary: fmt.Sprintf("the step %s has said nothing since %s%s", name, stamp(s.at), last),
|
|
Said: "no progress in " + w.progressBound.String(),
|
|
Headline: "A pipeline step has stopped saying it lives",
|
|
Explanation: "The step " + name + " started and has not ended, and neither it nor the run it reads has said its progress for " +
|
|
w.progressBound.String() + " or more" + last + ".",
|
|
Needs: "the part that runs it to say its progress, or the step ended (the runner's verbs mesh-pipeline-runner.stands and mesh-pipeline-runner.cancel)"}
|
|
if left := w.progressBound - w.now().Sub(s.at); left > 0 {
|
|
w.signal(stepRow, id, left, signalDue{Do: dueFind, Finding: f})
|
|
return
|
|
}
|
|
w.find(f)
|
|
}
|
|
|
|
// SeedSteps seeds the open steps from the runner's `stands` at the watchman's start (§731, A-468j): each step not already kept
|
|
// begins its clock now, and a step that waits on a person or for a build node is owed none.
|
|
func (w *Watchman) SeedSteps(stands []meshpipelinerunner.Wait) {
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
for _, s := range stands {
|
|
if s.Pipeline == "" || s.Step == "" {
|
|
continue
|
|
}
|
|
key := stepKey(s.Pipeline, s.Step, s.Artifact)
|
|
if _, ok := w.steps[key]; ok {
|
|
continue
|
|
}
|
|
owed := s.On != meshpipelinerunner.WaitOnPerson && s.On != meshpipelinerunner.WaitOnBuildNode && s.Who == ""
|
|
w.openStep(key, s.Pipeline, s.Step, s.Artifact, s.Ask, sharedAsks(s.Shared), w.now(), owed)
|
|
}
|
|
}
|