From ad40520e89a3add702e710cae1019e658a9c250c Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 17 Nov 2026 11:43:52 +0100 Subject: [PATCH] runner: stands names a waiting step's ask and its run, and every step asked says step-started (R6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The watchman could not time a step reading a run: stands named no ask, and step-started was said only on rerun and ask-again. A waiting step now stands with its own ask and, for a test step, the run's key and ask; each step asked at a start, a resume, a placing or a run says step-started, naming its run in shared with no asker. Builds: design 47 G §731, §729, §726 (step-started of a reading step); A-468e, A-469a, A-469q, A-469r, A-469v --- .../internal/pipeline/read.go | 18 ++++++- .../internal/pipeline/read_test.go | 53 +++++++++++++++++++ .../internal/pipeline/run_test.go | 8 ++- .../internal/pipeline/start.go | 42 +++++++++++++-- .../internal/pipeline/steps.go | 25 ++++++++- .../internal/pipeline/testruns_test.go | 8 +-- 6 files changed, 143 insertions(+), 11 deletions(-) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/read.go b/modules/mesh-delivery-pipeline/internal/pipeline/read.go index b244afbb..63d31d5a 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/read.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/read.go @@ -100,7 +100,7 @@ func (r Reads) Stands() (seat.StandsAnswer, error) { } out = append(out, seat.Wait{Pipeline: p.Pipeline, Module: p.Module, Branch: p.Branch, Commit: p.Commit, Group: p.Group, Run: p.Run, Source: p.Source, Revision: p.Revision, Fingerprint: p.Fingerprint, Step: s.Step, - Artifact: s.Artifact, On: s.On, Since: *s.Started, V: 1}) + Artifact: s.Artifact, On: s.On, Since: *s.Started, V: 1, Ask: askOfWait(s), Shared: sharedOfWait(s)}) } } sort.SliceStable(out, func(i, j int) bool { return out[i].Since.Before(out[j].Since) }) @@ -219,3 +219,19 @@ func startedOf(p seat.Pipeline) time.Time { } return *p.Started } + +// askOfWait is a waiting step's own ask (design 47 G §731, §496): a step reading another's test run keeps none (A-469m). +func askOfWait(s seat.StepResult) string { return s.Ask } + +// sharedOfWait is the run a waiting step waits on, where the runner knows it (§731; A-469r, A-469v): a test step's run key +// and ask, as one entry with no machine and no asker. +func sharedOfWait(s seat.StepResult) []seat.Shared { + if len(s.Shared) == 0 { + return nil + } + out := make([]seat.Shared, 0, len(s.Shared)) + for _, x := range s.Shared { + out = append(out, seat.Shared{Run: x.Run, Ask: x.Ask, Machine: x.Machine}) + } + return out +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/read_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/read_test.go index 20c544ab..8dad23b1 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/read_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/read_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" ) @@ -150,3 +151,55 @@ func TestShowAndKeptCarryTheStoresFill(t *testing.T) { t.Errorf("show with the fill unread: %v", err) } } + +// A step that reads a test run says its own step-started at every place it is asked, naming the run's key and ask in shared +// and no asker (design 47 G §726, §729; A-469a, A-469q), and stands names each waiting step's own ask and, for a test step, +// the run it waits on (§731; A-469r, A-469v). +func TestStandsAndStepStartedNameWhatAReadingStepWaitsOn(t *testing.T) { + f := newRunFixture() + p := f.start(t, 4, 9) + asker, reader := stepNamed(p, "app:program", "unit"), stepNamed(p, "app:tools", "unit") + started := map[string]seat.StepResult{} + for _, e := range f.said { + started[e.Result.Artifact+" "+e.Result.Step] = e.Result + } + for _, s := range []seat.StepResult{asker, reader} { + got, ok := started[s.Artifact+" unit"] + if !ok || len(got.Shared) != 1 || got.Shared[0] != (seat.Shared{Run: asker.Shared[0].Run, Ask: asker.Ask}) { + t.Errorf("the step-started of %s = %+v (said %v), want the run's key and ask, no asker", s.Artifact, got, ok) + } + } + if _, ok := started[" composition"]; !ok { + t.Error("the composition step's step-started is not said") + } + pipes := memKept{} + put(t, pipes, p.Pipeline, f.pipeline(p)) + a, err := Reads{Pipelines: pipes, Kept: memKept{}}.Stands() + if err != nil { + t.Fatal(err) + } + waits := map[string]seat.Wait{} + for _, w := range a.Stands { + waits[w.Artifact+" "+w.Step] = w + } + want := []seat.Shared{{Run: asker.Shared[0].Run, Ask: asker.Ask}} + if w := waits["app:program unit"]; w.Ask != asker.Ask || len(w.Shared) != 1 || w.Shared[0] != want[0] { + t.Errorf("the asking step stands as %+v, want its ask and its run", w) + } + if w := waits["app:tools unit"]; w.Ask != "" || len(w.Shared) != 1 || w.Shared[0] != want[0] { + t.Errorf("the reading step stands as %+v, want no ask of its own and the run's key and ask", w) + } + if w := waits[" composition"]; w.Ask == "" || len(w.Shared) != 0 { + t.Errorf("the composition step stands as %+v, want its own ask and no run", w) + } + // Resume asks the steps again and says their step-started again (A-469a). + n := len(f.said) + if _, err := f.s.Resume(f.store, "main", Half{Entries: map[string]catalog.CatalogEntry{ + "app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}}, + "mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}}}); err != nil { + t.Fatal(err) + } + if len(f.said) == n { + t.Error("resume said no step-started") + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/run_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/run_test.go index 6dbc0dc1..d2670bc5 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/run_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/run_test.go @@ -79,7 +79,13 @@ func TestRunOnRequest(t *testing.T) { if _, _, err := store.Get(RunKey("fix/1", a.Run, "same")); err == nil { t.Error("an entry the members do not change got a pipeline") } - if len(said) != 2 || !strings.HasPrefix(said[0], seat.EventPipelineStarted+".") || len(asks.ids) == 0 { + // Each pipeline placed says pipeline-started, then step-started for each step it asks (A-469a). + started, steps := 0, 0 + for _, x := range said { + started += strings.Count(x, seat.EventPipelineStarted+".") + steps += strings.Count(x, seat.EventStepStarted+".") + } + if started != 2 || !strings.HasPrefix(said[0], seat.EventPipelineStarted+".") || len(asks.ids) == 0 || steps < len(asks.ids) { t.Errorf("placing said %v and asked %d steps", said, len(asks.ids)) } if ids, _ := s.UnplacedRuns(); len(ids) != 0 { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index 449ce01b..e3916834 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -130,12 +130,46 @@ func (s *Starter) holderless(steps []seat.StepResult, asks []Asked, now time.Tim // askSteps puts a placed pipeline's asks (§22, §24): a test run another step asked is read, not asked again (share), then // the kept results and toolchain are carried and each ask is put. func (s *Starter) askSteps(key, commit string, asks []Asked) error { + placed := asks asks = s.share(key, asks) - if len(asks) == 0 { - return nil + var err error + if len(asks) > 0 { + s.carry(key, asks) + err = askAll(s.Steps, key, asks, s.recovered(key, commit)) + } + s.sayStarted(key, placed) + return err +} + +// sayStarted says step-started for each step of pipeline key that asks were placed for (design 47 G §726, §729; A-469a): +// the step asked, and each step reading the test run it asks, naming the run's key and ask in shared, so the watchman times +// it from that ask (§496, §732). A composition step's, said before any run is chosen, names none (§729). +func (s *Starter) sayStarted(key string, asks []Asked) { + if s.Store == nil || s.Say == nil || len(asks) == 0 { + return + } + ids, runs := map[string]bool{}, map[string]bool{} + for _, a := range asks { + ids[a.ID] = true + if a.Run != "" { + runs[a.Run] = true + } + } + p, _, err := s.Store.Get(key) + if err != nil { + s.logf("the pipeline %s is not read, so its steps' step-started is not said: %v", key, err) + return + } + at := s.now() + for _, st := range p.Steps { + if st.Outcome != seat.StepResultOutcomeRunning || !(ids[st.Ask] || (st.Ask == "" && runs[sharedRun(st)])) { + continue + } + subject, ev := stepEvent(seat.EventStepStarted, p, st, at) + if err := s.Say(subject, ev); err != nil { + s.logf("step-started of %s's %s %s is not said: %v", key, st.Artifact, st.Step, err) + } } - s.carry(key, asks) - return askAll(s.Steps, key, asks, s.recovered(key, commit)) } func newPipeline(b Begin, key string, now time.Time) seat.Pipeline { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index 4f32dbb1..c31f6639 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -432,8 +432,8 @@ func (h *Hearer) say(event string, p seat.Pipeline, step *seat.StepResult) error at := h.now() switch event { case seat.EventStepEnded, seat.EventStepStarted: - return h.Say(subject, seat.StepEvent{V: 1, At: &at, Pipeline: p.Pipeline, Branch: p.Branch, Module: p.Module, Commit: p.Commit, - Fingerprint: p.Fingerprint, Group: p.Group, Revision: p.Revision, Source: p.Source, Result: *step}) + _, ev := stepEvent(event, p, *step, at) + return h.Say(subject, ev) case seat.EventPipelineEnded: e := pipelineEnded(p) if h.Widens != nil { @@ -446,6 +446,27 @@ func (h *Hearer) say(event string, p seat.Pipeline, step *seat.StepResult) error } } +// stepEvent is a step's step-started or step-ended of p, on the runner's subject for it: its result as it stands, with +// its shared when it reads a run (§729; A-468e, A-469q). +func stepEvent(event string, p seat.Pipeline, step seat.StepResult, at time.Time) (string, seat.StepEvent) { + last := p.Commit + if p.Group != "" { + last = p.Group + } + if event == seat.EventStepStarted { + // A step-started names the run and its ask, never the step that asked it (A-469q). + var shared []seat.Shared + for _, x := range step.Shared { + x.AskedBy = nil + shared = append(shared, x) + } + step.Shared = shared + } + subject := "mesh.seat." + seat.Seat + ".event." + event + "." + keys.EscapeName(p.Module) + "." + keys.EscapeName(p.Branch) + "." + keys.EscapeName(last) + return subject, seat.StepEvent{V: 1, At: &at, Pipeline: p.Pipeline, Branch: p.Branch, Module: p.Module, Commit: p.Commit, + Fingerprint: p.Fingerprint, Group: p.Group, Revision: p.Revision, Source: p.Source, Result: step} +} + // The test budget per catalog module (design 47 G §24, §419; the test budget): at most 500 unit and 50 integration tests. const ( budgetUnit = 500 diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go index e80d6291..41185109 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go @@ -327,9 +327,11 @@ func TestARerunOfATestRunMovesEveryReaderThatDidNotPass(t *testing.T) { e.p.Steps[i].Outcome, e.p.Steps[i].Accepted = seat.StepResultOutcomeAccepted, &seat.Accepted{By: "op@shanks", Why: "known"} f.store[p.Pipeline] = e } - v := Verbs{Hearer: f.h, Starter: f.s, View: func(string) (Half, bool) { return Half{Entries: map[string]catalog.CatalogEntry{ - "app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}}, - "mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}}}, true }, Acts: &actLog{}} + v := Verbs{Hearer: f.h, Starter: f.s, View: func(string) (Half, bool) { + return Half{Entries: map[string]catalog.CatalogEntry{ + "app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}}, + "mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}}}, true + }, Acts: &actLog{}} a, err := v.Rerun("op@shanks", seat.RerunInput{Pipeline: p.Pipeline, Step: "unit", Artifact: "app:tools", Why: "flaky"}) if err != nil { t.Fatal(err)