G-24: one composition run per candidate and machine, one test run per level, every reading step timed #244
@@ -190,17 +190,23 @@ func brokerSubject(subject string) bool {
|
||||
return strings.HasPrefix(subject, "mesh.seat."+meshbroker.Seat+".event.")
|
||||
}
|
||||
|
||||
// pipelineSubject hands the gate's `source-moved` and the runner's `pipeline-started` to the watchman (watch/pipelines.go,
|
||||
// design 47 G §12): the head of a watched branch and its pipeline. It says whether the subject was one of them.
|
||||
// pipelineSubject hands the gate's `source-moved`, the runner's `pipeline-started`, `step-started` and `step-ended` and the assigner's
|
||||
// `step-progress` to the watchman (watch/pipelines.go, watch/steps.go, design 47 G §12, §732): the head of a watched branch, its
|
||||
// pipeline and each step's clock. It says whether the subject was one of them.
|
||||
func pipelineSubject(w *watch.Watchman, subject string, body []byte) bool {
|
||||
switch {
|
||||
case strings.HasPrefix(subject, "mesh.seat."+meshdelivery.Seat+".event."+meshdelivery.EventSourceMoved+"."):
|
||||
w.SourceMoved(body)
|
||||
case strings.HasPrefix(subject, "mesh.seat."+meshpipelinerunner.Seat+".event."+meshpipelinerunner.EventPipelineStarted+"."):
|
||||
w.PipelineStarted(body)
|
||||
case strings.HasPrefix(subject, "mesh.seat."+meshpipelinerunner.Seat+".event."+meshpipelinerunner.EventStepStarted+"."),
|
||||
strings.HasPrefix(subject, "mesh.seat."+meshpipelinerunner.Seat+".event."+meshpipelinerunner.EventStepEnded+"."):
|
||||
case strings.HasPrefix(subject, "mesh.seat."+meshpipelinerunner.Seat+".event."+meshpipelinerunner.EventStepStarted+"."):
|
||||
w.StepEvent(body)
|
||||
w.StepStarted(body)
|
||||
case strings.HasPrefix(subject, "mesh.seat."+meshpipelinerunner.Seat+".event."+meshpipelinerunner.EventStepEnded+"."):
|
||||
w.StepEvent(body)
|
||||
w.StepEnded(body)
|
||||
case strings.HasPrefix(subject, "mesh.seat."+meshwants.Seat+".event."+meshwants.EventStepProgress+"."):
|
||||
w.WantsStepProgress(body)
|
||||
default:
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
@@ -331,6 +332,31 @@ func TestWatchmanWhole(t *testing.T) { // 1135/AC8
|
||||
return finding(watch.FindingKey(watch.ScopeDelivery, stuckOf(askStream, ask1), watch.KindStuck))
|
||||
})
|
||||
until("node n1's silence", func() bool { return finding("machine.n1.silent") })
|
||||
|
||||
// Design 47 G §732 (test 108's watchman case): a composition step whose clock passed five minutes with no progress is
|
||||
// `pipeline-stands`; the assigner's `step-progress` for its ask clears it, and the step's own `step-ended` ends it.
|
||||
const pipeline, stepAsk = "main.9.web", "wants-ask-1"
|
||||
stepKey := pipeline + "/composition/"
|
||||
stepFinding := watch.FindingKey(watch.ScopeDelivery, watch.StepFindingID(stepKey), watch.KindPipelineStands)
|
||||
sayStep := func(event string, at time.Time) {
|
||||
t.Helper()
|
||||
body, _ := json.Marshal(meshpipelinerunner.StepEvent{Pipeline: pipeline, Branch: "main", Module: "web", Commit: "c1", At: &at,
|
||||
Result: meshpipelinerunner.StepResult{Step: "composition", Ask: stepAsk, Outcome: meshpipelinerunner.StepResultOutcomeRunning}})
|
||||
if _, err := js.Publish(ctx, "mesh.seat.mesh-pipeline-runner.event."+event+".web.main.c1", body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
sayStep("step-started", time.Now().UTC().Add(-6*time.Minute))
|
||||
until("the composition step with no progress for six minutes", func() bool { return finding(stepFinding) })
|
||||
progress, _ := json.Marshal(meshwants.StepProgressEvent{Ask: stepAsk, Asker: "runner", At: time.Now().UTC(), Phase: "run"})
|
||||
if _, err := js.Publish(ctx, "mesh.seat.mesh-wants.event.step-progress.n1", progress); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
until("the step's progress clears it", func() bool { return !finding(stepFinding) })
|
||||
sayStep("step-started", time.Now().UTC().Add(-6*time.Minute))
|
||||
until("the step stands again", func() bool { return finding(stepFinding) })
|
||||
sayStep("step-ended", time.Now().UTC())
|
||||
until("the step's end clears it", func() bool { return !finding(stepFinding) })
|
||||
// A-274: the finding is said under the watchman's binding finding-events.
|
||||
until("a finding-written under the binding finding-events", func() bool {
|
||||
st, err := js.Stream(ctx, "WATCHMAN_FINDINGS")
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"io"
|
||||
"log"
|
||||
@@ -400,6 +401,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
atCatalog := atHolder(calls.Call, holders, meshcatalog.Seat) // the indexer holds the seat mesh-catalog
|
||||
seedTakeIn(w, atCatalog, logger.Printf)
|
||||
seedVerdicts(w, atCatalog, logger.Printf)
|
||||
seedSteps(w, calls.Call, logger.Printf)
|
||||
calls.Close()
|
||||
}
|
||||
// The take-in events are the indexer's ordinary events (its emits), heard on the shared events stream through this
|
||||
@@ -507,6 +509,24 @@ func seedTakeIn(w *watch.Watchman, call func(key string, body any) (json.RawMess
|
||||
logf("take-in: %s cannot be asked (%v), so a wait the indexer held before this start is said only once a take-in event of it is heard", pendingTool, err)
|
||||
}
|
||||
|
||||
// standsVerb is the runner's seat verb that answers every step started and not ended (design 47 G §731, A-469p).
|
||||
const standsVerb = "seat:" + meshpipelinerunner.Seat + "." + meshpipelinerunner.VerbStands
|
||||
|
||||
// seedSteps asks the runner which steps stand now and keeps each, its clock begun at this start (Watchman.SeedSteps, design 47 G
|
||||
// §731, §732). A runner that does not answer is said: the steps are then known from the events heard after.
|
||||
func seedSteps(w *watch.Watchman, call func(key string, body any) (json.RawMessage, error), logf func(string, ...any)) {
|
||||
raw, err := call(standsVerb, map[string]any{})
|
||||
if err == nil {
|
||||
var a meshpipelinerunner.StandsAnswer
|
||||
if err = json.Unmarshal(raw, &a); err == nil {
|
||||
w.SeedSteps(a.Stands)
|
||||
logf("steps: %d open steps the runner holds are watched from this start (%s)", len(a.Stands), standsVerb)
|
||||
return
|
||||
}
|
||||
}
|
||||
logf("steps: %s cannot be asked (%v), so a step open before this start is timed only once an event of it is heard", standsVerb, err)
|
||||
}
|
||||
|
||||
// seedVerdicts asks the indexer's verdicts and raises the findings they hold (design 47 G §500): a verdict said while the watchman was
|
||||
// down is then not lost. An indexer that does not answer is said: the findings are then raised from the verdicts heard after.
|
||||
func seedVerdicts(w *watch.Watchman, call func(key string, body any) (json.RawMessage, error), logf func(string, ...any)) {
|
||||
|
||||
@@ -55,6 +55,9 @@ func (w *Watchman) StepProgress(seat, node string, body []byte) {
|
||||
w.say("a step-progress of %s does not read, left: %v", seat, err)
|
||||
return
|
||||
}
|
||||
w.mu.Lock()
|
||||
w.stepsHeardOf(e.Ask, e.At)
|
||||
w.mu.Unlock()
|
||||
w.askClock(seat, node, e.Ask, e.At)
|
||||
}
|
||||
|
||||
@@ -75,6 +78,7 @@ func (w *Watchman) AskEnded(seat string, body []byte) {
|
||||
}
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.stepsHeardOf(e.Ask, e.At)
|
||||
w.moved(bucket, e.Ask)
|
||||
}
|
||||
|
||||
|
||||
@@ -132,7 +132,7 @@ func (w *Watchman) StepEvent(body []byte) {
|
||||
w.say("a step event does not read, left: %v", err)
|
||||
return
|
||||
}
|
||||
id := e.Pipeline + "/" + e.Result.Step
|
||||
id := stepKey(e.Pipeline, e.Result.Step, e.Result.Artifact) // design 47 G A-469o
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
for seat, q := range w.queued {
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
package watch
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
)
|
||||
|
||||
// The watchman times every step that reads a run (design 47 G §731, §732, A-468j, A-469, A-469o, A-469p): each step started and
|
||||
// not ended is kept in memory under `<pipeline>/<step>/<artifact>` (the artifact empty for a module's step), its clock begun at its
|
||||
// `step-started` and begun again at each `mesh-wants` `step-progress` whose ask is the step's own ask or one its `shared` names, at
|
||||
// each build seat `step-progress` of such an ask, and at the `step-ended` of such an ask. A clock past the five-minute progress
|
||||
// bound raises `pipeline-stands`, cleared at the step's own `step-ended` or its next sign of life. A step waiting on a person or
|
||||
// queued for a build node (§86) is owed no progress. At its start the watchman seeds its open steps from the runner's `stands`.
|
||||
|
||||
// stepRow is the signals table's row of a step standing still.
|
||||
const stepRow = "step-stands"
|
||||
|
||||
// openStep is a step started and not ended.
|
||||
type openStep struct {
|
||||
pipeline, step, artifact string
|
||||
// asks are the step's own ask and the asks its shared runs name; ask the one last heard of.
|
||||
asks map[string]bool
|
||||
ask string
|
||||
// at is when the clock began; owed says whether the step is owed progress.
|
||||
at time.Time
|
||||
owed bool
|
||||
}
|
||||
|
||||
// stepKey is the key an open step is kept under.
|
||||
func stepKey(pipeline, step, artifact string) string { return pipeline + "/" + step + "/" + artifact }
|
||||
|
||||
// StepFindingID is the finding's id of an open step.
|
||||
func StepFindingID(key string) string { return "step-" + unsafeKey.ReplaceAllString(key, "_") }
|
||||
|
||||
// StepStarted hears the runner's `step-started`: the step is open and its clock begins at the event's time, unless it waits on a
|
||||
// person or for a build node, which are owed no progress. One that is no running or waiting step is ended.
|
||||
func (w *Watchman) StepStarted(body []byte) {
|
||||
var e meshpipelinerunner.StepEvent
|
||||
if err := json.Unmarshal(body, &e); err != nil || e.Pipeline == "" || e.Result.Step == "" {
|
||||
w.say("a step-started does not read, left: %v", err)
|
||||
return
|
||||
}
|
||||
key := stepKey(e.Pipeline, e.Result.Step, e.Result.Artifact)
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
switch e.Result.Outcome {
|
||||
case meshpipelinerunner.StepResultOutcomeRunning, "", meshpipelinerunner.StepResultOutcomeWaiting,
|
||||
meshpipelinerunner.StepResultOutcomeWaitingForABuildNode:
|
||||
default:
|
||||
w.closeStep(key)
|
||||
return
|
||||
}
|
||||
at := w.now()
|
||||
switch {
|
||||
case e.At != nil && !e.At.IsZero():
|
||||
at = *e.At
|
||||
case e.Result.Started != nil && !e.Result.Started.IsZero():
|
||||
at = *e.Result.Started
|
||||
}
|
||||
owed := e.Result.Outcome == meshpipelinerunner.StepResultOutcomeRunning || e.Result.Outcome == ""
|
||||
w.openStep(key, e.Pipeline, e.Result.Step, e.Result.Artifact, e.Result.Ask, sharedAsks(e.Result.Shared), at, owed)
|
||||
}
|
||||
|
||||
// StepEnded hears the runner's `step-ended`: the step's clock ends and its finding is cleared.
|
||||
func (w *Watchman) StepEnded(body []byte) {
|
||||
var e meshpipelinerunner.StepEvent
|
||||
if err := json.Unmarshal(body, &e); err != nil || e.Pipeline == "" || e.Result.Step == "" {
|
||||
w.say("a step-ended does not read, left: %v", err)
|
||||
return
|
||||
}
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.closeStep(stepKey(e.Pipeline, e.Result.Step, e.Result.Artifact))
|
||||
}
|
||||
|
||||
// WantsStepProgress hears the seat mesh-wants' `step-progress` (the assigner's, of a composition step): a step whose ask it is, or
|
||||
// a shared ask it names, is alive; the runs it names are kept as the step's.
|
||||
func (w *Watchman) WantsStepProgress(body []byte) {
|
||||
var e meshwants.StepProgressEvent
|
||||
if err := json.Unmarshal(body, &e); err != nil || e.Ask == "" {
|
||||
w.say("a step-progress of mesh-wants does not read, left: %v", err)
|
||||
return
|
||||
}
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
var named []string
|
||||
for _, s := range e.Shared {
|
||||
if s.Ask != "" {
|
||||
named = append(named, s.Ask)
|
||||
}
|
||||
}
|
||||
for key, s := range w.steps {
|
||||
if !s.asks[e.Ask] {
|
||||
continue
|
||||
}
|
||||
for _, a := range named {
|
||||
s.asks[a] = true
|
||||
}
|
||||
w.stepAlive(key, e.Ask, e.At)
|
||||
}
|
||||
}
|
||||
|
||||
// stepsHeardOf says the steps that name ask are alive: a build seat said its progress or its end (§732).
|
||||
func (w *Watchman) stepsHeardOf(ask string, at time.Time) {
|
||||
if ask == "" {
|
||||
return
|
||||
}
|
||||
for key, s := range w.steps {
|
||||
if s.asks[ask] {
|
||||
w.stepAlive(key, ask, at)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// stepAlive begins an owed step's clock again at at, never earlier than it began, w.mu held.
|
||||
func (w *Watchman) stepAlive(key, ask string, at time.Time) {
|
||||
s := w.steps[key]
|
||||
if at.IsZero() {
|
||||
at = w.now()
|
||||
}
|
||||
s.ask = ask
|
||||
if at.After(s.at) {
|
||||
s.at = at
|
||||
}
|
||||
w.timeStep(key)
|
||||
}
|
||||
|
||||
func sharedAsks(shared []meshpipelinerunner.Shared) []string {
|
||||
var out []string
|
||||
for _, s := range shared {
|
||||
if s.Ask != "" {
|
||||
out = append(out, s.Ask)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// openStep keeps a step and times it, w.mu held. A step kept before keeps the asks it knew.
|
||||
func (w *Watchman) openStep(key, pipeline, step, artifact, ask string, shared []string, at time.Time, owed bool) {
|
||||
s := w.steps[key]
|
||||
if s == nil {
|
||||
s = &openStep{pipeline: pipeline, step: step, artifact: artifact, asks: map[string]bool{}}
|
||||
w.steps[key] = s
|
||||
}
|
||||
s.at, s.owed = at, owed
|
||||
if ask != "" {
|
||||
s.asks[ask], s.ask = true, ask
|
||||
}
|
||||
for _, a := range shared {
|
||||
s.asks[a] = true
|
||||
}
|
||||
w.timeStep(key)
|
||||
}
|
||||
|
||||
// closeStep ends a step, w.mu held.
|
||||
func (w *Watchman) closeStep(key string) {
|
||||
if _, ok := w.steps[key]; !ok {
|
||||
return
|
||||
}
|
||||
delete(w.steps, key)
|
||||
id := StepFindingID(key)
|
||||
w.unsignal(stepRow, id)
|
||||
w.unfind(FindingKey(ScopeDelivery, id, KindPipelineStands))
|
||||
}
|
||||
|
||||
// timeStep times an open step, w.mu held: the progress bound from its clock, the finding raised at once when that has passed.
|
||||
func (w *Watchman) timeStep(key string) {
|
||||
s := w.steps[key]
|
||||
id := StepFindingID(key)
|
||||
w.unsignal(stepRow, id)
|
||||
w.unfind(FindingKey(ScopeDelivery, id, KindPipelineStands))
|
||||
if !s.owed {
|
||||
return
|
||||
}
|
||||
name := s.pipeline + " " + s.step
|
||||
if s.artifact != "" {
|
||||
name += " " + s.artifact
|
||||
}
|
||||
last := ""
|
||||
if s.ask != "" {
|
||||
last = ", last heard of its ask " + s.ask
|
||||
}
|
||||
f := Finding{Scope: ScopeDelivery, ID: id, Kind: KindPipelineStands, Severity: Warning, Source: stepRow, Since: s.at,
|
||||
Summary: fmt.Sprintf("the step %s has said nothing since %s%s", name, stamp(s.at), last),
|
||||
Said: "no progress in " + w.progressBound.String(),
|
||||
Headline: "A pipeline step has stopped saying it lives",
|
||||
Explanation: "The step " + name + " started and has not ended, and neither it nor the run it reads has said its progress for " +
|
||||
w.progressBound.String() + " or more" + last + ".",
|
||||
Needs: "the part that runs it to say its progress, or the step ended (the runner's pipeline_stands and runner_cancel tools)"}
|
||||
if left := w.progressBound - w.now().Sub(s.at); left > 0 {
|
||||
w.signal(stepRow, id, left, signalDue{Do: dueFind, Finding: f})
|
||||
return
|
||||
}
|
||||
w.find(f)
|
||||
}
|
||||
|
||||
// SeedSteps seeds the open steps from the runner's `stands` at the watchman's start (§731, A-468j): each step not already kept
|
||||
// begins its clock now, and a step that waits on a person or for a build node is owed none.
|
||||
func (w *Watchman) SeedSteps(stands []meshpipelinerunner.Wait) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
for _, s := range stands {
|
||||
if s.Pipeline == "" || s.Step == "" {
|
||||
continue
|
||||
}
|
||||
key := stepKey(s.Pipeline, s.Step, s.Artifact)
|
||||
if _, ok := w.steps[key]; ok {
|
||||
continue
|
||||
}
|
||||
owed := s.On != meshpipelinerunner.WaitOnPerson && s.On != meshpipelinerunner.WaitOnBuildNode && s.Who == ""
|
||||
w.openStep(key, s.Pipeline, s.Step, s.Artifact, s.Ask, sharedAsks(s.Shared), w.now(), owed)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
package watch
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
)
|
||||
|
||||
func stepBody(pipeline, step, artifact, ask, outcome string, at time.Time, shared ...string) []byte {
|
||||
var sh []meshpipelinerunner.Shared
|
||||
for _, a := range shared {
|
||||
sh = append(sh, meshpipelinerunner.Shared{Run: "r", Ask: a})
|
||||
}
|
||||
b, _ := json.Marshal(meshpipelinerunner.StepEvent{Pipeline: pipeline, Branch: "main", Module: "m", Commit: "c", At: &at,
|
||||
Result: meshpipelinerunner.StepResult{Step: step, Artifact: artifact, Ask: ask, Outcome: outcome, Shared: sh}})
|
||||
return b
|
||||
}
|
||||
|
||||
func wantsProgress(ask string, at time.Time, shared ...string) []byte {
|
||||
var sh []meshwants.Shared
|
||||
for _, a := range shared {
|
||||
sh = append(sh, meshwants.Shared{Machine: "n1", Run: "r", Ask: a})
|
||||
}
|
||||
b, _ := json.Marshal(meshwants.StepProgressEvent{Ask: ask, Asker: "a", At: at, Phase: "run", Shared: sh})
|
||||
return b
|
||||
}
|
||||
|
||||
// design 47 G §731, §732, A-468j, A-469, A-469o, A-469p (test 108's and 109's watchman case): a step started and not ended is
|
||||
// timed from its `step-started` and from every sign of life of its own ask or an ask its `shared` names, raises `pipeline-stands`
|
||||
// at the five-minute bound and clears at its `step-ended`; a step on a person or queued for a build node is owed none; the open
|
||||
// steps are seeded from `stands`.
|
||||
func TestAStepThatReadsARunIsTimed(t *testing.T) {
|
||||
run := meshpipelinerunner.StepResultOutcomeRunning
|
||||
key := func(pipeline, step, artifact string) string {
|
||||
return signalKey(stepRow, StepFindingID(stepKey(pipeline, step, artifact)))
|
||||
}
|
||||
finding := func(pipeline, step, artifact string) string {
|
||||
return FindingKey(ScopeDelivery, StepFindingID(stepKey(pipeline, step, artifact)), KindPipelineStands)
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
run func(w *Watchman, o *fakeOut, c *clock)
|
||||
// stands are the finding keys that stand at the end.
|
||||
stands []string
|
||||
}{
|
||||
{"no progress for five minutes", func(w *Watchman, o *fakeOut, c *clock) {
|
||||
w.StepStarted(stepBody("p", "composition", "", "a1", run, c.t))
|
||||
if o.ttls[key("p", "composition", "")] != ProgressBound || len(o.findings) != 0 {
|
||||
t.Fatalf("within the bound: %v %v", o.ttls, o.findings)
|
||||
}
|
||||
expire(w, c, key("p", "composition", ""), ProgressBound)
|
||||
}, []string{finding("p", "composition", "")}},
|
||||
{"the assigner's progress of its ask begins the clock again, and names the runs", func(w *Watchman, o *fakeOut, c *clock) {
|
||||
w.StepStarted(stepBody("p", "composition", "", "a1", run, c.t))
|
||||
c.t = c.t.Add(4 * time.Minute)
|
||||
w.WantsStepProgress(wantsProgress("a1", c.t, "run1"))
|
||||
if o.ttls[key("p", "composition", "")] != ProgressBound {
|
||||
t.Fatalf("clock not begun again: %v", o.ttls)
|
||||
}
|
||||
c.t = c.t.Add(4 * time.Minute)
|
||||
w.StepProgress("node-build-go", "n1", askBody("run1", c.t)) // an ask the progress named
|
||||
if o.ttls[key("p", "composition", "")] != ProgressBound {
|
||||
t.Fatalf("a named run's progress is the step's: %v", o.ttls)
|
||||
}
|
||||
w.WantsStepProgress(wantsProgress("other", c.t))
|
||||
}, nil},
|
||||
{"the progress of a shared run's ask, and its end, begin a test step's clock again", func(w *Watchman, o *fakeOut, c *clock) {
|
||||
w.StepStarted(stepBody("p", "unit", "m:program", "a2", run, c.t, "run-ask"))
|
||||
c.t = c.t.Add(4 * time.Minute)
|
||||
w.StepProgress("node-build-go", "n1", askBody("run-ask", c.t))
|
||||
c.t = c.t.Add(4 * time.Minute)
|
||||
w.AskEnded("node-build-go", askBody("run-ask", c.t))
|
||||
expire(w, c, key("p", "unit", "m:program"), ProgressBound)
|
||||
}, []string{finding("p", "unit", "m:program")}},
|
||||
{"the artifact keys the step: the other artifact's end clears only its own", func(w *Watchman, o *fakeOut, c *clock) {
|
||||
w.StepStarted(stepBody("p", "unit", "m:program", "a2", run, c.t))
|
||||
w.StepStarted(stepBody("p", "unit", "m:tools", "a3", run, c.t))
|
||||
c.t = c.t.Add(ProgressBound)
|
||||
w.Pulse(key("p", "unit", "m:program"), Expired, c.t)
|
||||
w.Pulse(key("p", "unit", "m:tools"), Expired, c.t)
|
||||
w.StepEnded(stepBody("p", "unit", "m:tools", "a3", meshpipelinerunner.StepResultOutcomePassed, c.t))
|
||||
}, []string{finding("p", "unit", "m:program")}},
|
||||
{"its step-ended clears the finding", func(w *Watchman, o *fakeOut, c *clock) {
|
||||
w.StepStarted(stepBody("p", "composition", "", "a1", run, c.t))
|
||||
expire(w, c, key("p", "composition", ""), ProgressBound)
|
||||
w.StepEnded(stepBody("p", "composition", "", "a1", meshpipelinerunner.StepResultOutcomeFailed, c.t))
|
||||
}, nil},
|
||||
{"a step on a person or queued for a build node is owed none", func(w *Watchman, o *fakeOut, c *clock) {
|
||||
w.StepStarted(stepBody("p", "approval", "", "", meshpipelinerunner.StepResultOutcomeWaiting, c.t))
|
||||
w.StepStarted(stepBody("p", "unit", "m:program", "a2", meshpipelinerunner.StepResultOutcomeWaitingForABuildNode, c.t))
|
||||
c.t = c.t.Add(time.Hour)
|
||||
if len(o.pulses) != 0 || len(o.findings) != 0 {
|
||||
t.Fatalf("owed none: %v %v", o.pulses, o.findings)
|
||||
}
|
||||
w.StepStarted(stepBody("p", "unit", "m:program", "a2", run, c.t)) // taken: now owed
|
||||
expire(w, c, key("p", "unit", "m:program"), ProgressBound)
|
||||
}, []string{finding("p", "unit", "m:program")}},
|
||||
{"seeded from stands: each clock begins at the start, a step on a person is owed none", func(w *Watchman, o *fakeOut, c *clock) {
|
||||
w.SeedSteps([]meshpipelinerunner.Wait{
|
||||
{Pipeline: "p", Step: "composition", Ask: "a1", On: ""},
|
||||
{Pipeline: "p", Step: "unit", Artifact: "m:program", Shared: []meshpipelinerunner.Shared{{Run: "r", Ask: "run-ask"}}},
|
||||
{Pipeline: "p", Step: "approval", On: meshpipelinerunner.WaitOnPerson},
|
||||
})
|
||||
if len(o.findings) != 0 || o.ttls[key("p", "composition", "")] != ProgressBound || len(o.pulses) != 2 {
|
||||
t.Fatalf("seeded at the start: %v %v", o.ttls, o.findings)
|
||||
}
|
||||
c.t = c.t.Add(4 * time.Minute)
|
||||
w.WantsStepProgress(wantsProgress("a1", c.t)) // the composition step lives
|
||||
expire(w, c, key("p", "unit", "m:program"), time.Minute)
|
||||
}, []string{finding("p", "unit", "m:program")}},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
w, o, c := setup()
|
||||
w.SetStepBound(ProgressBound) // setup shortens it for the older tests
|
||||
tc.run(w, o, c)
|
||||
if len(o.findings) != len(tc.stands) {
|
||||
t.Fatalf("findings %v, want %v", o.findings, tc.stands)
|
||||
}
|
||||
for _, k := range tc.stands {
|
||||
if _, ok := o.findings[k]; !ok {
|
||||
t.Fatalf("findings %v, want %s", o.findings, k)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -121,6 +121,8 @@ type Watchman struct {
|
||||
recordMilestone map[string]string
|
||||
// queued is, by build seat, each step queued on it for a build node and since when (pipelines.go).
|
||||
queued map[string]map[string]time.Time
|
||||
// steps is each pipeline step started and not ended, by `<pipeline>/<step>/<artifact>` (steps.go, design 47 G §732).
|
||||
steps map[string]*openStep
|
||||
// awaySaid is each machine whose machine-away was said and not yet its machine-back, with when it was last heard (presence.go).
|
||||
awaySaid map[string]time.Time
|
||||
// opened is each record past its bound and not ended, with its finding (records.go).
|
||||
@@ -135,7 +137,7 @@ func New(module string, out Out, now func() time.Time) *Watchman {
|
||||
advised: map[string]int{}, advisedSince: map[string]time.Time{},
|
||||
provFailed: map[string]map[string]ProvReport{}, closures: map[string]map[string]time.Time{}, waiting: map[string]map[string]string{},
|
||||
|
||||
standing: map[string]map[string]bool{}, readWhole: map[string]bool{}, awaited: map[string]bool{}, heads: map[string]head{}, queued: map[string]map[string]time.Time{}, awaySaid: map[string]time.Time{}, opened: map[string]Finding{}}
|
||||
standing: map[string]map[string]bool{}, readWhole: map[string]bool{}, awaited: map[string]bool{}, heads: map[string]head{}, queued: map[string]map[string]time.Time{}, steps: map[string]*openStep{}, awaySaid: map[string]time.Time{}, opened: map[string]Finding{}}
|
||||
}
|
||||
|
||||
// Logf sets where a write that failed is said; it is silent until set.
|
||||
|
||||
@@ -127,11 +127,11 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6
|
||||
}
|
||||
// The join: its first view asked again by hand is a hand act, recorded through the seat (views.StatusTools). The waits the
|
||||
// indexer holds at the watchman's start are asked of its tool (ADR 0633 §5), a verb edge as the view's.
|
||||
if want := []string{"mesh-delivery-indexer.indexer_pending", "mesh-delivery-indexer.indexer_verdicts", "seat:mesh-hand-acts.hand-act", "seat:mesh-conditions.conditions", "seat:mesh-broker.show", "seat:mesh-catalog.revision", "seat:mesh-catalog.registrations", "seat:mesh-delivery.source"}; !reflect.DeepEqual(m.Invokes, want) {
|
||||
if want := []string{"mesh-delivery-indexer.indexer_pending", "mesh-delivery-indexer.indexer_verdicts", "seat:mesh-hand-acts.hand-act", "seat:mesh-conditions.conditions", "seat:mesh-broker.show", "seat:mesh-catalog.revision", "seat:mesh-catalog.registrations", "seat:mesh-delivery.source", "seat:mesh-pipeline-runner.stands"}; !reflect.DeepEqual(m.Invokes, want) {
|
||||
t.Errorf("invokes = %v, want %v", m.Invokes, want)
|
||||
}
|
||||
// DLV-M5 its first views, 1585/T20 the indexer's take-in events, and DLV-W1 (ADR 0595 §4): the accepted verdicts and the bus module's created and refused, durably.
|
||||
if want := []string{"seat:mesh-wants.views.wants", "seat:mesh-module-assigner.views.seat-holders", "mesh-catalog.registration-changed", "mesh-catalog.catalog-revision", "mesh-delivery-indexer.take-in.pending", "mesh-delivery-indexer.take-in.taken", "mesh-catalog.manifest-accepted", "mesh-catalog.manifest-refused", "mesh-delivery.source-moved", "mesh-delivery.filling", "mesh-delivery.full", "mesh-delivery.room", "mesh-delivery.record-opened", "mesh-delivery.record-ended", "mesh-catalog.filling", "mesh-catalog.full", "mesh-catalog.room", "mesh-wants.filling", "mesh-wants.full", "mesh-wants.room", "mesh-grants.filling", "mesh-grants.full", "mesh-grants.room", "node-build-go.ask-taken", "node-build-go.step-progress", "node-build-go.step-ended", "node-build-image.ask-taken", "node-build-image.step-progress", "node-build-image.step-ended", "node-build-node.ask-taken", "node-build-node.step-progress", "node-build-node.step-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.step-started", "mesh-pipeline-runner.step-ended", "mesh-pipeline-runner.wait-held", "mesh-pipeline-runner.wait-ended", "mesh-wants.wait-held", "mesh-wants.wait-ended", "mesh-grants.wait-held", "mesh-grants.wait-ended", "mesh-delivery-builder.wait-held", "mesh-delivery-builder.wait-ended", "node-engine.wait-held", "node-engine.wait-ended", "node-engine.running", "mesh-broker.created", "mesh-broker.refused", "mesh-broker.recreated", "mesh-broker.dropped", "mesh-conditions.findings-asked"}; !reflect.DeepEqual(m.Consumes, want) {
|
||||
if want := []string{"seat:mesh-wants.views.wants", "seat:mesh-module-assigner.views.seat-holders", "mesh-catalog.registration-changed", "mesh-catalog.catalog-revision", "mesh-delivery-indexer.take-in.pending", "mesh-delivery-indexer.take-in.taken", "mesh-catalog.manifest-accepted", "mesh-catalog.manifest-refused", "mesh-delivery.source-moved", "mesh-delivery.filling", "mesh-delivery.full", "mesh-delivery.room", "mesh-delivery.record-opened", "mesh-delivery.record-ended", "mesh-catalog.filling", "mesh-catalog.full", "mesh-catalog.room", "mesh-wants.filling", "mesh-wants.full", "mesh-wants.room", "mesh-grants.filling", "mesh-grants.full", "mesh-grants.room", "node-build-go.ask-taken", "node-build-go.step-progress", "node-build-go.step-ended", "node-build-image.ask-taken", "node-build-image.step-progress", "node-build-image.step-ended", "node-build-node.ask-taken", "node-build-node.step-progress", "node-build-node.step-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.step-started", "mesh-pipeline-runner.step-ended", "mesh-wants.step-progress", "mesh-pipeline-runner.wait-held", "mesh-pipeline-runner.wait-ended", "mesh-wants.wait-held", "mesh-wants.wait-ended", "mesh-grants.wait-held", "mesh-grants.wait-ended", "mesh-delivery-builder.wait-held", "mesh-delivery-builder.wait-ended", "node-engine.wait-held", "node-engine.wait-ended", "node-engine.running", "mesh-broker.created", "mesh-broker.refused", "mesh-broker.recreated", "mesh-broker.dropped", "mesh-conditions.findings-asked"}; !reflect.DeepEqual(m.Consumes, want) {
|
||||
t.Errorf("consumes = %v, want %v", m.Consumes, want)
|
||||
}
|
||||
// 1952/WM1: the watchman holds the seat mesh-watchdogs at mesh scope, its first seat (novox/hq ADR 0564 §1, §4).
|
||||
|
||||
@@ -26,7 +26,8 @@
|
||||
"seat:mesh-broker.show",
|
||||
"seat:mesh-catalog.revision",
|
||||
"seat:mesh-catalog.registrations",
|
||||
"seat:mesh-delivery.source"
|
||||
"seat:mesh-delivery.source",
|
||||
"seat:mesh-pipeline-runner.stands"
|
||||
],
|
||||
"consumes": [
|
||||
"seat:mesh-wants.views.wants",
|
||||
@@ -64,6 +65,7 @@
|
||||
"mesh-pipeline-runner.pipeline-started",
|
||||
"mesh-pipeline-runner.step-started",
|
||||
"mesh-pipeline-runner.step-ended",
|
||||
"mesh-wants.step-progress",
|
||||
"mesh-pipeline-runner.wait-held",
|
||||
"mesh-pipeline-runner.wait-ended",
|
||||
"mesh-wants.wait-held",
|
||||
|
||||
Reference in New Issue
Block a user