From ebe49b9bdd496010e749959638abc0b2573cb17f Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 17 Nov 2026 11:48:28 +0100 Subject: [PATCH] runner: a test run's steps are said waiting for a build node until the holder takes the run's ask (A-469b) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A test step queued on a build seat was said running, so the watchman's build-capacity waits never saw it; every step of a test run now waits for a build node from the run's ask, and at the holder's ask-taken each is said started again, running. Builds: design 47 G §725, §732, §86; A-469b --- .../internal/pipeline/approval.go | 2 +- .../internal/pipeline/place.go | 5 ++- .../internal/pipeline/place_test.go | 2 +- .../internal/pipeline/read.go | 2 +- .../internal/pipeline/runends.go | 44 ++++++++++++++++++- .../internal/pipeline/start.go | 6 +-- .../internal/pipeline/steps.go | 6 +-- .../internal/pipeline/supersede.go | 4 +- .../internal/pipeline/testruns.go | 5 ++- .../internal/pipeline/testruns_test.go | 13 ++++++ .../internal/pipeline/verbs.go | 8 +++- 11 files changed, 80 insertions(+), 17 deletions(-) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/approval.go b/modules/mesh-delivery-pipeline/internal/pipeline/approval.go index beeff984..c3d32346 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/approval.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/approval.go @@ -38,7 +38,7 @@ func approvalDue(p seat.Pipeline) int { } continue } - if s.Outcome == seat.StepResultOutcomeWaiting || s.Outcome == seat.StepResultOutcomeRunning { + if s.Outcome == seat.StepResultOutcomeWaiting || open(s) { return -1 } if s.Status == seat.StepResultStatusRequired && s.Outcome != seat.StepResultOutcomePassed && s.Outcome != seat.StepResultOutcomeCarried { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place.go b/modules/mesh-delivery-pipeline/internal/pipeline/place.go index 698e19b7..a9bec3c3 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place.go @@ -139,12 +139,13 @@ func place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog // Another artifact's step of this pipeline asks the run: this one reads it. first := &asks[j] first.Readers = append(first.Readers, reader) - s.Ask = "" + s.Ask, s.Outcome, s.On = "", seat.StepResultOutcomeWaitingForABuildNode, seat.WaitOnBuildNode s.Shared = []seat.Shared{{Run: run, Ask: first.ID, AskedBy: &seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: first.Readers[0].Artifact}}} steps = append(steps, s) continue } - s.Shared = []seat.Shared{{Run: run, Ask: s.Ask}} + // A test run's steps wait for a build node from its ask until the holder's ask-taken (A-469b). + s.Shared, s.Outcome, s.On = []seat.Shared{{Run: run, Ask: s.Ask}}, seat.StepResultOutcomeWaitingForABuildNode, seat.WaitOnBuildNode ask.Run, ask.Level, ask.Digest, ask.Readers = run, step, digest, []seat.TestRunReader{reader} askOfRun[run] = len(asks) } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go index f516c158..eb49aa9a 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go @@ -407,7 +407,7 @@ func TestATestRunIsAskedOncePerCandidateLevelAndDigest(t *testing.T) { if asker.Ask == "" || len(asker.Shared) != 1 || asker.Shared[0].Ask != asker.Ask || asker.Shared[0].AskedBy != nil { t.Errorf("the asking %s step = %+v, want its ask named in shared with no asked-by", level, asker) } - if reader.Ask != "" || reader.Outcome != seat.StepResultOutcomeRunning || len(reader.Shared) != 1 || reader.Shared[0] != (seat.Shared{ + if reader.Ask != "" || reader.Outcome != seat.StepResultOutcomeWaitingForABuildNode || reader.On != seat.WaitOnBuildNode || len(reader.Shared) != 1 || reader.Shared[0] != (seat.Shared{ Run: asker.Shared[0].Run, Ask: asker.Ask, AskedBy: reader.Shared[0].AskedBy}) || reader.Shared[0].AskedBy == nil || *reader.Shared[0].AskedBy != (seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: "app:program"}) { t.Errorf("the reading %s step = %+v, want no ask and the program's run named", level, reader) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/read.go b/modules/mesh-delivery-pipeline/internal/pipeline/read.go index 63d31d5a..9fed9e40 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/read.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/read.go @@ -95,7 +95,7 @@ func (r Reads) Stands() (seat.StandsAnswer, error) { if s.Ended != nil || s.Started == nil { continue } - if s.Outcome != seat.StepResultOutcomeRunning && s.Outcome != seat.StepResultOutcomeWaiting { + if !open(s) && s.Outcome != seat.StepResultOutcomeWaiting { continue } out = append(out, seat.Wait{Pipeline: p.Pipeline, Module: p.Module, Branch: p.Branch, Commit: p.Commit, diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/runends.go b/modules/mesh-delivery-pipeline/internal/pipeline/runends.go index 8d98ba7e..122936ae 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/runends.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/runends.go @@ -116,6 +116,7 @@ func (h *Hearer) RunTaken(ask, node string) error { return fmt.Errorf("the test run %s is not written: %w", run, err) } } + h.runStarted(run, r) break } if err := h.Clock.Bound(BoundPrefix+run, h.runSettings().StepBound); err != nil { @@ -296,7 +297,7 @@ func (h *Hearer) endReader(reader RunReader, run string, r Run) error { if s.Artifact != reader.Artifact || sharedRun(s) != run { continue } - if s.Outcome == seat.StepResultOutcomeRunning { + if open(s) { i, moved = j, false } else if reader.Rerun && s.Shared[0].Ask != r.Ask && s.Accepted == nil { i, moved = j, true @@ -392,3 +393,44 @@ func (h *Hearer) leaveRun(key string, s seat.StepResult, why string) { return } } + +// open says whether a step is asked and not ended: running, or a test step waiting for a build node to take its run's ask +// (A-469b). +func open(s seat.StepResult) bool { + return s.Outcome == seat.StepResultOutcomeRunning || s.Outcome == seat.StepResultOutcomeWaitingForABuildNode +} + +// runStarted moves each step reading run from waiting for a build node to running, the holder's ask-taken heard (A-469b), +// and says its step-started again, so the watchman's build-capacity waits let it go (§86, §732). +func (h *Hearer) runStarted(run string, r Run) { + for _, reader := range r.Readers { + for { + p, rev, err := h.Store.Get(reader.Pipeline) + if err != nil { + break + } + var moved []int + for j, s := range p.Steps { + if s.Artifact == reader.Artifact && sharedRun(s) == run && s.Outcome == seat.StepResultOutcomeWaitingForABuildNode { + p.Steps[j].Outcome, p.Steps[j].On = seat.StepResultOutcomeRunning, "" + moved = append(moved, j) + } + } + if len(moved) == 0 { + break + } + if _, err := h.Store.Update(reader.Pipeline, p, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + h.logf("the steps of %s reading the taken test run %s are not written running: %v", reader.Pipeline, run, err) + break + } + for _, j := range moved { + if err := h.say(seat.EventStepStarted, p, &p.Steps[j]); err != nil { + h.logf("step-started of %s's %s is not said: %v", reader.Pipeline, p.Steps[j].Step, err) + } + } + break + } + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index e3916834..7a6617df 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -105,7 +105,7 @@ func (s *Starter) holderless(steps []seat.StepResult, asks []Asked, now time.Tim none := map[string]bool{} for i := range steps { st := &steps[i] - if st.Outcome != seat.StepResultOutcomeRunning || !BoundSeat(st.Seat) { + if !open(*st) || !BoundSeat(st.Seat) { continue } if holders, read := s.Holders(st.Seat); !read || len(holders) > 0 { @@ -162,7 +162,7 @@ func (s *Starter) sayStarted(key string, asks []Asked) { } at := s.now() for _, st := range p.Steps { - if st.Outcome != seat.StepResultOutcomeRunning || !(ids[st.Ask] || (st.Ask == "" && runs[sharedRun(st)])) { + if !open(st) || !(ids[st.Ask] || (st.Ask == "" && runs[sharedRun(st)])) { continue } subject, ev := stepEvent(seat.EventStepStarted, p, st, at) @@ -278,7 +278,7 @@ func (s *Starter) Resume(store Updater, branch string, half Half) (int, error) { // runs and keeps its own ask, ending by the path it was asked under (§730, A-468k). running, own := map[string]bool{}, map[string]bool{} for _, st := range p.Steps { - if st.Outcome == seat.StepResultOutcomeRunning && st.Ask != "" { + if open(st) && st.Ask != "" { running[st.Ask], own[st.Ask] = true, len(st.Shared) == 0 } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index db8f1f1c..7f2989ee 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -201,7 +201,7 @@ func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, sh case ev.Commit != p.Commit || ev.Fingerprint != p.Fingerprint || ev.Module != p.Module || ev.Step != p.Steps[i].Step: return p, fmt.Errorf("%w: ask %s ended %s of %s at %s (%s), asked as %s at %s (%s)", ErrRefused, ev.ID, ev.Step, ev.Module, ev.Commit, ev.Fingerprint, p.Steps[i].Step, p.Commit, p.Fingerprint) - case p.Steps[i].Outcome != seat.StepResultOutcomeRunning: + case !open(p.Steps[i]): if run := sharedRun(p.Steps[i]); run != "" && h.TestRuns != nil { if r, _, err := h.TestRuns.Get(run); err == nil && r.Ended == nil && r.Ask == ev.ID { // The asking step's end was written and its run's was not (a write failed): the run ends now, heard again. @@ -310,7 +310,7 @@ func (h *Hearer) sameRunReaders(p *seat.Pipeline, i int, ev StepEnded, now time. continue } switch m := marks[s.Artifact]; { - case s.Outcome == seat.StepResultOutcomeRunning && s.Ask == "": + case open(*s) && s.Ask == "": case m.Rerun && s.Shared[0].Ask != ev.ID && s.Accepted == nil: s.Rerun, s.By = true, m.By s.Shared = []seat.Shared{{Run: run, Ask: ev.ID, AskedBy: &seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: asker.Artifact}}} @@ -550,7 +550,7 @@ func endPipeline(p *seat.Pipeline, now time.Time) bool { // An optional step still waiting (a code review nobody has given) holds nothing: only a required one holds the // pipeline's end (A-104). continue - case s.Outcome == seat.StepResultOutcomeWaiting, s.Outcome == seat.StepResultOutcomeRunning: + case s.Outcome == seat.StepResultOutcomeWaiting, open(s): return false } if s.Status == seat.StepResultStatusRequired && s.Outcome != seat.StepResultOutcomePassed && s.Outcome != seat.StepResultOutcomeCarried { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go b/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go index f008149d..30be4d2f 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/supersede.go @@ -30,7 +30,7 @@ func (h *Hearer) Taken(ask, node string) error { break } i := stepOfAsk(p, ask) - if i < 0 || p.Steps[i].Outcome != seat.StepResultOutcomeRunning || p.Steps[i].Node == node { + if i < 0 || !open(p.Steps[i]) || p.Steps[i].Node == node { break } p.Steps[i].Node = node @@ -110,7 +110,7 @@ func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop var changed []int for i := range p.Steps { s := &p.Steps[i] - if s.Outcome != seat.StepResultOutcomeRunning && s.Outcome != seat.StepResultOutcomeWaiting { + if !open(*s) && s.Outcome != seat.StepResultOutcomeWaiting { continue } // A step of a test run leaves it: its ask is dropped only with its last reader (§726, leaveRun). diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go index be004716..46ea8e7c 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns.go @@ -321,9 +321,12 @@ func (s *Starter) readOthers(key string, joined map[string]Run) { for i := range p.Steps { st := &p.Steps[i] r, ok := joined[sharedRun(*st)] - if !ok || st.Outcome != seat.StepResultOutcomeRunning { + if !ok || !open(*st) { continue } + if r.Node != "" { + st.Outcome, st.On = seat.StepResultOutcomeRunning, "" // its run's ask is taken already (A-469b) + } st.Ask = "" st.Shared = []seat.Shared{{Run: sharedRun(*st), Ask: r.Ask, AskedBy: by[sharedRun(*st)]}} } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go index f37047d2..b57f2352 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go @@ -114,9 +114,22 @@ func TestATestRunHasOneClockACapAndAnAge(t *testing.T) { if f.clock.bound[UntakenPrefix+run] != 24*time.Hour || len(f.clock.bound) != 3 { t.Fatalf("bounds at the start = %v, want the three runs' untaken bounds of 24h", f.clock.bound) } + // A-469b: the run's steps wait for a build node until the holder's ask-taken, then run, each said started again. + if unit.Outcome != seat.StepResultOutcomeWaitingForABuildNode || stepNamed(p, "app:tools", "unit").Outcome != seat.StepResultOutcomeWaitingForABuildNode { + t.Fatalf("the unit steps before ask-taken = %s, %s", unit.Outcome, stepNamed(p, "app:tools", "unit").Outcome) + } + n := len(f.said) if err := f.h.RunTaken(unit.Ask, "ace"); err != nil { t.Fatal(err) } + for _, art := range []string{"app:program", "app:tools"} { + if s := stepNamed(f.pipeline(p), art, "unit"); s.Outcome != seat.StepResultOutcomeRunning { + t.Errorf("the %s unit step after ask-taken is %s, want running", art, s.Outcome) + } + } + if len(f.said)-n != 2 || f.said[n].Result.Outcome != seat.StepResultOutcomeRunning { + t.Errorf("ask-taken said %d step events, want the two unit steps started running", len(f.said)-n) + } if r, _, _ := f.runs.Get(run); r.Node != "ace" || f.clock.bound[BoundPrefix+run] != 30*time.Minute { t.Fatalf("after ask-taken the run = %+v, bounds %v; want its node and its clock of 30m", r, f.clock.bound) } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go index 554bf8f3..1608d60c 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go @@ -206,7 +206,7 @@ func (v Verbs) Cancel(caller string, in seat.CancelInput) (seat.CancelAnswer, er var leave []seat.StepResult for i := range p.Steps { s := &p.Steps[i] - if s.Outcome != seat.StepResultOutcomeRunning && s.Outcome != seat.StepResultOutcomeWaiting { + if !open(*s) && s.Outcome != seat.StepResultOutcomeWaiting { continue } if sharedRun(*s) != "" && s.Seat != wants.Seat { @@ -306,7 +306,7 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult) (seat.RerunAnswer, error) { h := v.Hearer run := sharedRun(st) - if st.Outcome == seat.StepResultOutcomeRunning { + if open(st) { return seat.RerunAnswer{}, fmt.Errorf("the %s step of %s is running: it is not asked again", in.Step, in.Pipeline) } r, rev, err := h.TestRuns.Get(run) @@ -357,6 +357,7 @@ func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult) switch { case ask != nil: s.Ask, s.Shared = ask.ID, []seat.Shared{{Run: run, Ask: ask.ID}} + s.Outcome, s.On = seat.StepResultOutcomeWaitingForABuildNode, seat.WaitOnBuildNode // until ask-taken (A-469b) case read: s.Shared = []seat.Shared{{Run: run, Ask: r.Ask, AskedBy: asker}} s.Outcome, s.Why, s.Node, s.Ended = r.Outcome, r.Said, r.Node, &now @@ -366,6 +367,9 @@ func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult) } default: s.Shared = []seat.Shared{{Run: run, Ask: r.Ask, AskedBy: asker}} + if r.Node == "" { + s.Outcome, s.On = seat.StepResultOutcomeWaitingForABuildNode, seat.WaitOnBuildNode + } } if p.Outcome != seat.PipelineOutcomeRunning { p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeRunning, "", nil