diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go index fd878ccc..895cfae4 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go @@ -190,17 +190,23 @@ func brokerSubject(subject string) bool { return strings.HasPrefix(subject, "mesh.seat."+meshbroker.Seat+".event.") } -// pipelineSubject hands the gate's `source-moved` and the runner's `pipeline-started` to the watchman (watch/pipelines.go, -// design 47 G §12): the head of a watched branch and its pipeline. It says whether the subject was one of them. +// pipelineSubject hands the gate's `source-moved`, the runner's `pipeline-started`, `step-started` and `step-ended` and the assigner's +// `step-progress` to the watchman (watch/pipelines.go, watch/steps.go, design 47 G §12, §732): the head of a watched branch, its +// pipeline and each step's clock. It says whether the subject was one of them. func pipelineSubject(w *watch.Watchman, subject string, body []byte) bool { switch { case strings.HasPrefix(subject, "mesh.seat."+meshdelivery.Seat+".event."+meshdelivery.EventSourceMoved+"."): w.SourceMoved(body) case strings.HasPrefix(subject, "mesh.seat."+meshpipelinerunner.Seat+".event."+meshpipelinerunner.EventPipelineStarted+"."): w.PipelineStarted(body) - case strings.HasPrefix(subject, "mesh.seat."+meshpipelinerunner.Seat+".event."+meshpipelinerunner.EventStepStarted+"."), - strings.HasPrefix(subject, "mesh.seat."+meshpipelinerunner.Seat+".event."+meshpipelinerunner.EventStepEnded+"."): + case strings.HasPrefix(subject, "mesh.seat."+meshpipelinerunner.Seat+".event."+meshpipelinerunner.EventStepStarted+"."): w.StepEvent(body) + w.StepStarted(body) + case strings.HasPrefix(subject, "mesh.seat."+meshpipelinerunner.Seat+".event."+meshpipelinerunner.EventStepEnded+"."): + w.StepEvent(body) + w.StepEnded(body) + case strings.HasPrefix(subject, "mesh.seat."+meshwants.Seat+".event."+meshwants.EventStepProgress+"."): + w.WantsStepProgress(body) default: return false } diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/integration_test.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/integration_test.go index b73bd86f..ae028d0b 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/integration_test.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/integration_test.go @@ -22,6 +22,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-sdk/go/buscompose" @@ -331,6 +332,31 @@ func TestWatchmanWhole(t *testing.T) { // 1135/AC8 return finding(watch.FindingKey(watch.ScopeDelivery, stuckOf(askStream, ask1), watch.KindStuck)) }) until("node n1's silence", func() bool { return finding("machine.n1.silent") }) + + // Design 47 G §732 (test 108's watchman case): a composition step whose clock passed five minutes with no progress is + // `pipeline-stands`; the assigner's `step-progress` for its ask clears it, and the step's own `step-ended` ends it. + const pipeline, stepAsk = "main.9.web", "wants-ask-1" + stepKey := pipeline + "/composition/" + stepFinding := watch.FindingKey(watch.ScopeDelivery, watch.StepFindingID(stepKey), watch.KindPipelineStands) + sayStep := func(event string, at time.Time) { + t.Helper() + body, _ := json.Marshal(meshpipelinerunner.StepEvent{Pipeline: pipeline, Branch: "main", Module: "web", Commit: "c1", At: &at, + Result: meshpipelinerunner.StepResult{Step: "composition", Ask: stepAsk, Outcome: meshpipelinerunner.StepResultOutcomeRunning}}) + if _, err := js.Publish(ctx, "mesh.seat.mesh-pipeline-runner.event."+event+".web.main.c1", body); err != nil { + t.Fatal(err) + } + } + sayStep("step-started", time.Now().UTC().Add(-6*time.Minute)) + until("the composition step with no progress for six minutes", func() bool { return finding(stepFinding) }) + progress, _ := json.Marshal(meshwants.StepProgressEvent{Ask: stepAsk, Asker: "runner", At: time.Now().UTC(), Phase: "run"}) + if _, err := js.Publish(ctx, "mesh.seat.mesh-wants.event.step-progress.n1", progress); err != nil { + t.Fatal(err) + } + until("the step's progress clears it", func() bool { return !finding(stepFinding) }) + sayStep("step-started", time.Now().UTC().Add(-6*time.Minute)) + until("the step stands again", func() bool { return finding(stepFinding) }) + sayStep("step-ended", time.Now().UTC()) + until("the step's end clears it", func() bool { return !finding(stepFinding) }) // A-274: the finding is said under the watchman's binding finding-events. until("a finding-written under the binding finding-events", func() bool { st, err := js.Stream(ctx, "WATCHMAN_FINDINGS") diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go index a39e2c55..06174855 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go @@ -13,6 +13,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "io" "log" @@ -400,6 +401,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro atCatalog := atHolder(calls.Call, holders, meshcatalog.Seat) // the indexer holds the seat mesh-catalog seedTakeIn(w, atCatalog, logger.Printf) seedVerdicts(w, atCatalog, logger.Printf) + seedSteps(w, calls.Call, logger.Printf) calls.Close() } // The take-in events are the indexer's ordinary events (its emits), heard on the shared events stream through this @@ -507,6 +509,24 @@ func seedTakeIn(w *watch.Watchman, call func(key string, body any) (json.RawMess logf("take-in: %s cannot be asked (%v), so a wait the indexer held before this start is said only once a take-in event of it is heard", pendingTool, err) } +// standsVerb is the runner's seat verb that answers every step started and not ended (design 47 G §731, A-469p). +const standsVerb = "seat:" + meshpipelinerunner.Seat + "." + meshpipelinerunner.VerbStands + +// seedSteps asks the runner which steps stand now and keeps each, its clock begun at this start (Watchman.SeedSteps, design 47 G +// §731, §732). A runner that does not answer is said: the steps are then known from the events heard after. +func seedSteps(w *watch.Watchman, call func(key string, body any) (json.RawMessage, error), logf func(string, ...any)) { + raw, err := call(standsVerb, map[string]any{}) + if err == nil { + var a meshpipelinerunner.StandsAnswer + if err = json.Unmarshal(raw, &a); err == nil { + w.SeedSteps(a.Stands) + logf("steps: %d open steps the runner holds are watched from this start (%s)", len(a.Stands), standsVerb) + return + } + } + logf("steps: %s cannot be asked (%v), so a step open before this start is timed only once an event of it is heard", standsVerb, err) +} + // seedVerdicts asks the indexer's verdicts and raises the findings they hold (design 47 G §500): a verdict said while the watchman was // down is then not lost. An indexer that does not answer is said: the findings are then raised from the verdicts heard after. func seedVerdicts(w *watch.Watchman, call func(key string, body any) (json.RawMessage, error), logf func(string, ...any)) { diff --git a/modules/mesh-delivery-watchman/internal/watch/mirror.go b/modules/mesh-delivery-watchman/internal/watch/mirror.go index afd564e6..af76a4b1 100644 --- a/modules/mesh-delivery-watchman/internal/watch/mirror.go +++ b/modules/mesh-delivery-watchman/internal/watch/mirror.go @@ -55,6 +55,9 @@ func (w *Watchman) StepProgress(seat, node string, body []byte) { w.say("a step-progress of %s does not read, left: %v", seat, err) return } + w.mu.Lock() + w.stepsHeardOf(e.Ask, e.At) + w.mu.Unlock() w.askClock(seat, node, e.Ask, e.At) } @@ -75,6 +78,7 @@ func (w *Watchman) AskEnded(seat string, body []byte) { } w.mu.Lock() defer w.mu.Unlock() + w.stepsHeardOf(e.Ask, e.At) w.moved(bucket, e.Ask) } diff --git a/modules/mesh-delivery-watchman/internal/watch/pipelines.go b/modules/mesh-delivery-watchman/internal/watch/pipelines.go index 09324691..11faf479 100644 --- a/modules/mesh-delivery-watchman/internal/watch/pipelines.go +++ b/modules/mesh-delivery-watchman/internal/watch/pipelines.go @@ -132,7 +132,7 @@ func (w *Watchman) StepEvent(body []byte) { w.say("a step event does not read, left: %v", err) return } - id := e.Pipeline + "/" + e.Result.Step + id := stepKey(e.Pipeline, e.Result.Step, e.Result.Artifact) // design 47 G A-469o w.mu.Lock() defer w.mu.Unlock() for seat, q := range w.queued { diff --git a/modules/mesh-delivery-watchman/internal/watch/steps.go b/modules/mesh-delivery-watchman/internal/watch/steps.go new file mode 100644 index 00000000..9b9d0ee6 --- /dev/null +++ b/modules/mesh-delivery-watchman/internal/watch/steps.go @@ -0,0 +1,217 @@ +package watch + +import ( + "encoding/json" + "fmt" + "time" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" +) + +// The watchman times every step that reads a run (design 47 G §731, §732, A-468j, A-469, A-469o, A-469p): each step started and +// not ended is kept in memory under `//` (the artifact empty for a module's step), its clock begun at its +// `step-started` and begun again at each `mesh-wants` `step-progress` whose ask is the step's own ask or one its `shared` names, at +// each build seat `step-progress` of such an ask, and at the `step-ended` of such an ask. A clock past the five-minute progress +// bound raises `pipeline-stands`, cleared at the step's own `step-ended` or its next sign of life. A step waiting on a person or +// queued for a build node (§86) is owed no progress. At its start the watchman seeds its open steps from the runner's `stands`. + +// stepRow is the signals table's row of a step standing still. +const stepRow = "step-stands" + +// openStep is a step started and not ended. +type openStep struct { + pipeline, step, artifact string + // asks are the step's own ask and the asks its shared runs name; ask the one last heard of. + asks map[string]bool + ask string + // at is when the clock began; owed says whether the step is owed progress. + at time.Time + owed bool +} + +// stepKey is the key an open step is kept under. +func stepKey(pipeline, step, artifact string) string { return pipeline + "/" + step + "/" + artifact } + +// StepFindingID is the finding's id of an open step. +func StepFindingID(key string) string { return "step-" + unsafeKey.ReplaceAllString(key, "_") } + +// StepStarted hears the runner's `step-started`: the step is open and its clock begins at the event's time, unless it waits on a +// person or for a build node, which are owed no progress. One that is no running or waiting step is ended. +func (w *Watchman) StepStarted(body []byte) { + var e meshpipelinerunner.StepEvent + if err := json.Unmarshal(body, &e); err != nil || e.Pipeline == "" || e.Result.Step == "" { + w.say("a step-started does not read, left: %v", err) + return + } + key := stepKey(e.Pipeline, e.Result.Step, e.Result.Artifact) + w.mu.Lock() + defer w.mu.Unlock() + switch e.Result.Outcome { + case meshpipelinerunner.StepResultOutcomeRunning, "", meshpipelinerunner.StepResultOutcomeWaiting, + meshpipelinerunner.StepResultOutcomeWaitingForABuildNode: + default: + w.closeStep(key) + return + } + at := w.now() + switch { + case e.At != nil && !e.At.IsZero(): + at = *e.At + case e.Result.Started != nil && !e.Result.Started.IsZero(): + at = *e.Result.Started + } + owed := e.Result.Outcome == meshpipelinerunner.StepResultOutcomeRunning || e.Result.Outcome == "" + w.openStep(key, e.Pipeline, e.Result.Step, e.Result.Artifact, e.Result.Ask, sharedAsks(e.Result.Shared), at, owed) +} + +// StepEnded hears the runner's `step-ended`: the step's clock ends and its finding is cleared. +func (w *Watchman) StepEnded(body []byte) { + var e meshpipelinerunner.StepEvent + if err := json.Unmarshal(body, &e); err != nil || e.Pipeline == "" || e.Result.Step == "" { + w.say("a step-ended does not read, left: %v", err) + return + } + w.mu.Lock() + defer w.mu.Unlock() + w.closeStep(stepKey(e.Pipeline, e.Result.Step, e.Result.Artifact)) +} + +// WantsStepProgress hears the seat mesh-wants' `step-progress` (the assigner's, of a composition step): a step whose ask it is, or +// a shared ask it names, is alive; the runs it names are kept as the step's. +func (w *Watchman) WantsStepProgress(body []byte) { + var e meshwants.StepProgressEvent + if err := json.Unmarshal(body, &e); err != nil || e.Ask == "" { + w.say("a step-progress of mesh-wants does not read, left: %v", err) + return + } + w.mu.Lock() + defer w.mu.Unlock() + var named []string + for _, s := range e.Shared { + if s.Ask != "" { + named = append(named, s.Ask) + } + } + for key, s := range w.steps { + if !s.asks[e.Ask] { + continue + } + for _, a := range named { + s.asks[a] = true + } + w.stepAlive(key, e.Ask, e.At) + } +} + +// stepsHeardOf says the steps that name ask are alive: a build seat said its progress or its end (§732). +func (w *Watchman) stepsHeardOf(ask string, at time.Time) { + if ask == "" { + return + } + for key, s := range w.steps { + if s.asks[ask] { + w.stepAlive(key, ask, at) + } + } +} + +// stepAlive begins an owed step's clock again at at, never earlier than it began, w.mu held. +func (w *Watchman) stepAlive(key, ask string, at time.Time) { + s := w.steps[key] + if at.IsZero() { + at = w.now() + } + s.ask = ask + if at.After(s.at) { + s.at = at + } + w.timeStep(key) +} + +func sharedAsks(shared []meshpipelinerunner.Shared) []string { + var out []string + for _, s := range shared { + if s.Ask != "" { + out = append(out, s.Ask) + } + } + return out +} + +// openStep keeps a step and times it, w.mu held. A step kept before keeps the asks it knew. +func (w *Watchman) openStep(key, pipeline, step, artifact, ask string, shared []string, at time.Time, owed bool) { + s := w.steps[key] + if s == nil { + s = &openStep{pipeline: pipeline, step: step, artifact: artifact, asks: map[string]bool{}} + w.steps[key] = s + } + s.at, s.owed = at, owed + if ask != "" { + s.asks[ask], s.ask = true, ask + } + for _, a := range shared { + s.asks[a] = true + } + w.timeStep(key) +} + +// closeStep ends a step, w.mu held. +func (w *Watchman) closeStep(key string) { + if _, ok := w.steps[key]; !ok { + return + } + delete(w.steps, key) + id := StepFindingID(key) + w.unsignal(stepRow, id) + w.unfind(FindingKey(ScopeDelivery, id, KindPipelineStands)) +} + +// timeStep times an open step, w.mu held: the progress bound from its clock, the finding raised at once when that has passed. +func (w *Watchman) timeStep(key string) { + s := w.steps[key] + id := StepFindingID(key) + w.unsignal(stepRow, id) + w.unfind(FindingKey(ScopeDelivery, id, KindPipelineStands)) + if !s.owed { + return + } + name := s.pipeline + " " + s.step + if s.artifact != "" { + name += " " + s.artifact + } + last := "" + if s.ask != "" { + last = ", last heard of its ask " + s.ask + } + f := Finding{Scope: ScopeDelivery, ID: id, Kind: KindPipelineStands, Severity: Warning, Source: stepRow, Since: s.at, + Summary: fmt.Sprintf("the step %s has said nothing since %s%s", name, stamp(s.at), last), + Said: "no progress in " + w.progressBound.String(), + Headline: "A pipeline step has stopped saying it lives", + Explanation: "The step " + name + " started and has not ended, and neither it nor the run it reads has said its progress for " + + w.progressBound.String() + " or more" + last + ".", + Needs: "the part that runs it to say its progress, or the step ended (the runner's pipeline_stands and runner_cancel tools)"} + if left := w.progressBound - w.now().Sub(s.at); left > 0 { + w.signal(stepRow, id, left, signalDue{Do: dueFind, Finding: f}) + return + } + w.find(f) +} + +// SeedSteps seeds the open steps from the runner's `stands` at the watchman's start (§731, A-468j): each step not already kept +// begins its clock now, and a step that waits on a person or for a build node is owed none. +func (w *Watchman) SeedSteps(stands []meshpipelinerunner.Wait) { + w.mu.Lock() + defer w.mu.Unlock() + for _, s := range stands { + if s.Pipeline == "" || s.Step == "" { + continue + } + key := stepKey(s.Pipeline, s.Step, s.Artifact) + if _, ok := w.steps[key]; ok { + continue + } + owed := s.On != meshpipelinerunner.WaitOnPerson && s.On != meshpipelinerunner.WaitOnBuildNode && s.Who == "" + w.openStep(key, s.Pipeline, s.Step, s.Artifact, s.Ask, sharedAsks(s.Shared), w.now(), owed) + } +} diff --git a/modules/mesh-delivery-watchman/internal/watch/steps_test.go b/modules/mesh-delivery-watchman/internal/watch/steps_test.go new file mode 100644 index 00000000..41320f5a --- /dev/null +++ b/modules/mesh-delivery-watchman/internal/watch/steps_test.go @@ -0,0 +1,130 @@ +package watch + +import ( + "encoding/json" + "testing" + "time" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" +) + +func stepBody(pipeline, step, artifact, ask, outcome string, at time.Time, shared ...string) []byte { + var sh []meshpipelinerunner.Shared + for _, a := range shared { + sh = append(sh, meshpipelinerunner.Shared{Run: "r", Ask: a}) + } + b, _ := json.Marshal(meshpipelinerunner.StepEvent{Pipeline: pipeline, Branch: "main", Module: "m", Commit: "c", At: &at, + Result: meshpipelinerunner.StepResult{Step: step, Artifact: artifact, Ask: ask, Outcome: outcome, Shared: sh}}) + return b +} + +func wantsProgress(ask string, at time.Time, shared ...string) []byte { + var sh []meshwants.Shared + for _, a := range shared { + sh = append(sh, meshwants.Shared{Machine: "n1", Run: "r", Ask: a}) + } + b, _ := json.Marshal(meshwants.StepProgressEvent{Ask: ask, Asker: "a", At: at, Phase: "run", Shared: sh}) + return b +} + +// design 47 G §731, §732, A-468j, A-469, A-469o, A-469p (test 108's and 109's watchman case): a step started and not ended is +// timed from its `step-started` and from every sign of life of its own ask or an ask its `shared` names, raises `pipeline-stands` +// at the five-minute bound and clears at its `step-ended`; a step on a person or queued for a build node is owed none; the open +// steps are seeded from `stands`. +func TestAStepThatReadsARunIsTimed(t *testing.T) { + run := meshpipelinerunner.StepResultOutcomeRunning + key := func(pipeline, step, artifact string) string { + return signalKey(stepRow, StepFindingID(stepKey(pipeline, step, artifact))) + } + finding := func(pipeline, step, artifact string) string { + return FindingKey(ScopeDelivery, StepFindingID(stepKey(pipeline, step, artifact)), KindPipelineStands) + } + cases := []struct { + name string + run func(w *Watchman, o *fakeOut, c *clock) + // stands are the finding keys that stand at the end. + stands []string + }{ + {"no progress for five minutes", func(w *Watchman, o *fakeOut, c *clock) { + w.StepStarted(stepBody("p", "composition", "", "a1", run, c.t)) + if o.ttls[key("p", "composition", "")] != ProgressBound || len(o.findings) != 0 { + t.Fatalf("within the bound: %v %v", o.ttls, o.findings) + } + expire(w, c, key("p", "composition", ""), ProgressBound) + }, []string{finding("p", "composition", "")}}, + {"the assigner's progress of its ask begins the clock again, and names the runs", func(w *Watchman, o *fakeOut, c *clock) { + w.StepStarted(stepBody("p", "composition", "", "a1", run, c.t)) + c.t = c.t.Add(4 * time.Minute) + w.WantsStepProgress(wantsProgress("a1", c.t, "run1")) + if o.ttls[key("p", "composition", "")] != ProgressBound { + t.Fatalf("clock not begun again: %v", o.ttls) + } + c.t = c.t.Add(4 * time.Minute) + w.StepProgress("node-build-go", "n1", askBody("run1", c.t)) // an ask the progress named + if o.ttls[key("p", "composition", "")] != ProgressBound { + t.Fatalf("a named run's progress is the step's: %v", o.ttls) + } + w.WantsStepProgress(wantsProgress("other", c.t)) + }, nil}, + {"the progress of a shared run's ask, and its end, begin a test step's clock again", func(w *Watchman, o *fakeOut, c *clock) { + w.StepStarted(stepBody("p", "unit", "m:program", "a2", run, c.t, "run-ask")) + c.t = c.t.Add(4 * time.Minute) + w.StepProgress("node-build-go", "n1", askBody("run-ask", c.t)) + c.t = c.t.Add(4 * time.Minute) + w.AskEnded("node-build-go", askBody("run-ask", c.t)) + expire(w, c, key("p", "unit", "m:program"), ProgressBound) + }, []string{finding("p", "unit", "m:program")}}, + {"the artifact keys the step: the other artifact's end clears only its own", func(w *Watchman, o *fakeOut, c *clock) { + w.StepStarted(stepBody("p", "unit", "m:program", "a2", run, c.t)) + w.StepStarted(stepBody("p", "unit", "m:tools", "a3", run, c.t)) + c.t = c.t.Add(ProgressBound) + w.Pulse(key("p", "unit", "m:program"), Expired, c.t) + w.Pulse(key("p", "unit", "m:tools"), Expired, c.t) + w.StepEnded(stepBody("p", "unit", "m:tools", "a3", meshpipelinerunner.StepResultOutcomePassed, c.t)) + }, []string{finding("p", "unit", "m:program")}}, + {"its step-ended clears the finding", func(w *Watchman, o *fakeOut, c *clock) { + w.StepStarted(stepBody("p", "composition", "", "a1", run, c.t)) + expire(w, c, key("p", "composition", ""), ProgressBound) + w.StepEnded(stepBody("p", "composition", "", "a1", meshpipelinerunner.StepResultOutcomeFailed, c.t)) + }, nil}, + {"a step on a person or queued for a build node is owed none", func(w *Watchman, o *fakeOut, c *clock) { + w.StepStarted(stepBody("p", "approval", "", "", meshpipelinerunner.StepResultOutcomeWaiting, c.t)) + w.StepStarted(stepBody("p", "unit", "m:program", "a2", meshpipelinerunner.StepResultOutcomeWaitingForABuildNode, c.t)) + c.t = c.t.Add(time.Hour) + if len(o.pulses) != 0 || len(o.findings) != 0 { + t.Fatalf("owed none: %v %v", o.pulses, o.findings) + } + w.StepStarted(stepBody("p", "unit", "m:program", "a2", run, c.t)) // taken: now owed + expire(w, c, key("p", "unit", "m:program"), ProgressBound) + }, []string{finding("p", "unit", "m:program")}}, + {"seeded from stands: each clock begins at the start, a step on a person is owed none", func(w *Watchman, o *fakeOut, c *clock) { + w.SeedSteps([]meshpipelinerunner.Wait{ + {Pipeline: "p", Step: "composition", Ask: "a1", On: ""}, + {Pipeline: "p", Step: "unit", Artifact: "m:program", Shared: []meshpipelinerunner.Shared{{Run: "r", Ask: "run-ask"}}}, + {Pipeline: "p", Step: "approval", On: meshpipelinerunner.WaitOnPerson}, + }) + if len(o.findings) != 0 || o.ttls[key("p", "composition", "")] != ProgressBound || len(o.pulses) != 2 { + t.Fatalf("seeded at the start: %v %v", o.ttls, o.findings) + } + c.t = c.t.Add(4 * time.Minute) + w.WantsStepProgress(wantsProgress("a1", c.t)) // the composition step lives + expire(w, c, key("p", "unit", "m:program"), time.Minute) + }, []string{finding("p", "unit", "m:program")}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + w, o, c := setup() + w.SetStepBound(ProgressBound) // setup shortens it for the older tests + tc.run(w, o, c) + if len(o.findings) != len(tc.stands) { + t.Fatalf("findings %v, want %v", o.findings, tc.stands) + } + for _, k := range tc.stands { + if _, ok := o.findings[k]; !ok { + t.Fatalf("findings %v, want %s", o.findings, k) + } + } + }) + } +} diff --git a/modules/mesh-delivery-watchman/internal/watch/watch.go b/modules/mesh-delivery-watchman/internal/watch/watch.go index 43b15745..f65d75b2 100644 --- a/modules/mesh-delivery-watchman/internal/watch/watch.go +++ b/modules/mesh-delivery-watchman/internal/watch/watch.go @@ -121,6 +121,8 @@ type Watchman struct { recordMilestone map[string]string // queued is, by build seat, each step queued on it for a build node and since when (pipelines.go). queued map[string]map[string]time.Time + // steps is each pipeline step started and not ended, by `//` (steps.go, design 47 G §732). + steps map[string]*openStep // awaySaid is each machine whose machine-away was said and not yet its machine-back, with when it was last heard (presence.go). awaySaid map[string]time.Time // opened is each record past its bound and not ended, with its finding (records.go). @@ -135,7 +137,7 @@ func New(module string, out Out, now func() time.Time) *Watchman { advised: map[string]int{}, advisedSince: map[string]time.Time{}, provFailed: map[string]map[string]ProvReport{}, closures: map[string]map[string]time.Time{}, waiting: map[string]map[string]string{}, - standing: map[string]map[string]bool{}, readWhole: map[string]bool{}, awaited: map[string]bool{}, heads: map[string]head{}, queued: map[string]map[string]time.Time{}, awaySaid: map[string]time.Time{}, opened: map[string]Finding{}} + standing: map[string]map[string]bool{}, readWhole: map[string]bool{}, awaited: map[string]bool{}, heads: map[string]head{}, queued: map[string]map[string]time.Time{}, steps: map[string]*openStep{}, awaySaid: map[string]time.Time{}, opened: map[string]Finding{}} } // Logf sets where a write that failed is said; it is silent until set. diff --git a/modules/mesh-delivery-watchman/manifest_test.go b/modules/mesh-delivery-watchman/manifest_test.go index 197709a7..0e53eb8a 100644 --- a/modules/mesh-delivery-watchman/manifest_test.go +++ b/modules/mesh-delivery-watchman/manifest_test.go @@ -127,11 +127,11 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6 } // The join: its first view asked again by hand is a hand act, recorded through the seat (views.StatusTools). The waits the // indexer holds at the watchman's start are asked of its tool (ADR 0633 §5), a verb edge as the view's. - if want := []string{"mesh-delivery-indexer.indexer_pending", "mesh-delivery-indexer.indexer_verdicts", "seat:mesh-hand-acts.hand-act", "seat:mesh-conditions.conditions", "seat:mesh-broker.show", "seat:mesh-catalog.revision", "seat:mesh-catalog.registrations", "seat:mesh-delivery.source"}; !reflect.DeepEqual(m.Invokes, want) { + if want := []string{"mesh-delivery-indexer.indexer_pending", "mesh-delivery-indexer.indexer_verdicts", "seat:mesh-hand-acts.hand-act", "seat:mesh-conditions.conditions", "seat:mesh-broker.show", "seat:mesh-catalog.revision", "seat:mesh-catalog.registrations", "seat:mesh-delivery.source", "seat:mesh-pipeline-runner.stands"}; !reflect.DeepEqual(m.Invokes, want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } // DLV-M5 its first views, 1585/T20 the indexer's take-in events, and DLV-W1 (ADR 0595 §4): the accepted verdicts and the bus module's created and refused, durably. - if want := []string{"seat:mesh-wants.views.wants", "seat:mesh-module-assigner.views.seat-holders", "mesh-catalog.registration-changed", "mesh-catalog.catalog-revision", "mesh-delivery-indexer.take-in.pending", "mesh-delivery-indexer.take-in.taken", "mesh-catalog.manifest-accepted", "mesh-catalog.manifest-refused", "mesh-delivery.source-moved", "mesh-delivery.filling", "mesh-delivery.full", "mesh-delivery.room", "mesh-delivery.record-opened", "mesh-delivery.record-ended", "mesh-catalog.filling", "mesh-catalog.full", "mesh-catalog.room", "mesh-wants.filling", "mesh-wants.full", "mesh-wants.room", "mesh-grants.filling", "mesh-grants.full", "mesh-grants.room", "node-build-go.ask-taken", "node-build-go.step-progress", "node-build-go.step-ended", "node-build-image.ask-taken", "node-build-image.step-progress", "node-build-image.step-ended", "node-build-node.ask-taken", "node-build-node.step-progress", "node-build-node.step-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.step-started", "mesh-pipeline-runner.step-ended", "mesh-pipeline-runner.wait-held", "mesh-pipeline-runner.wait-ended", "mesh-wants.wait-held", "mesh-wants.wait-ended", "mesh-grants.wait-held", "mesh-grants.wait-ended", "mesh-delivery-builder.wait-held", "mesh-delivery-builder.wait-ended", "node-engine.wait-held", "node-engine.wait-ended", "node-engine.running", "mesh-broker.created", "mesh-broker.refused", "mesh-broker.recreated", "mesh-broker.dropped", "mesh-conditions.findings-asked"}; !reflect.DeepEqual(m.Consumes, want) { + if want := []string{"seat:mesh-wants.views.wants", "seat:mesh-module-assigner.views.seat-holders", "mesh-catalog.registration-changed", "mesh-catalog.catalog-revision", "mesh-delivery-indexer.take-in.pending", "mesh-delivery-indexer.take-in.taken", "mesh-catalog.manifest-accepted", "mesh-catalog.manifest-refused", "mesh-delivery.source-moved", "mesh-delivery.filling", "mesh-delivery.full", "mesh-delivery.room", "mesh-delivery.record-opened", "mesh-delivery.record-ended", "mesh-catalog.filling", "mesh-catalog.full", "mesh-catalog.room", "mesh-wants.filling", "mesh-wants.full", "mesh-wants.room", "mesh-grants.filling", "mesh-grants.full", "mesh-grants.room", "node-build-go.ask-taken", "node-build-go.step-progress", "node-build-go.step-ended", "node-build-image.ask-taken", "node-build-image.step-progress", "node-build-image.step-ended", "node-build-node.ask-taken", "node-build-node.step-progress", "node-build-node.step-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.step-started", "mesh-pipeline-runner.step-ended", "mesh-wants.step-progress", "mesh-pipeline-runner.wait-held", "mesh-pipeline-runner.wait-ended", "mesh-wants.wait-held", "mesh-wants.wait-ended", "mesh-grants.wait-held", "mesh-grants.wait-ended", "mesh-delivery-builder.wait-held", "mesh-delivery-builder.wait-ended", "node-engine.wait-held", "node-engine.wait-ended", "node-engine.running", "mesh-broker.created", "mesh-broker.refused", "mesh-broker.recreated", "mesh-broker.dropped", "mesh-conditions.findings-asked"}; !reflect.DeepEqual(m.Consumes, want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } // 1952/WM1: the watchman holds the seat mesh-watchdogs at mesh scope, its first seat (novox/hq ADR 0564 §1, §4). diff --git a/modules/mesh-delivery-watchman/module.json b/modules/mesh-delivery-watchman/module.json index 29bfff97..b3078253 100644 --- a/modules/mesh-delivery-watchman/module.json +++ b/modules/mesh-delivery-watchman/module.json @@ -26,7 +26,8 @@ "seat:mesh-broker.show", "seat:mesh-catalog.revision", "seat:mesh-catalog.registrations", - "seat:mesh-delivery.source" + "seat:mesh-delivery.source", + "seat:mesh-pipeline-runner.stands" ], "consumes": [ "seat:mesh-wants.views.wants", @@ -64,6 +65,7 @@ "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.step-started", "mesh-pipeline-runner.step-ended", + "mesh-wants.step-progress", "mesh-pipeline-runner.wait-held", "mesh-pipeline-runner.wait-ended", "mesh-wants.wait-held",