From 773f89e9de9979e6eafab0037bd2fd258914332b Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 15 Nov 2026 18:33:37 +0100 Subject: [PATCH] steprun: the build seats' queue kinds take turns A build module pulled a waiting delivery before any check, so 89 queued deliveries held every pipeline's check back on shanks for an hour and more. The pull now starts at the kind after the one its last ask came from; neither kind starves the other (the operator's yes, 2026-11-15). --- modules/mesh-delivery-sdk/steprun/pull.go | 19 +++++++++--- .../mesh-delivery-sdk/steprun/pull_test.go | 29 +++++++++++++++++++ 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/modules/mesh-delivery-sdk/steprun/pull.go b/modules/mesh-delivery-sdk/steprun/pull.go index a4c167b7..8da13c6f 100644 --- a/modules/mesh-delivery-sdk/steprun/pull.go +++ b/modules/mesh-delivery-sdk/steprun/pull.go @@ -169,6 +169,8 @@ type Puller struct { // declined are the kinds whose worker handed an ask this node declined, left out of the pulls without a wait until // the next pull's wait ends. declined map[string]bool + // turn is the kind the next pull asks first (next): the kinds take turns, so a queue that never empties holds back none. + turn int } func (p *Puller) init() { @@ -519,21 +521,30 @@ func (p *Puller) Pull(ctx context.Context) error { } } -// next is one ask from the workers in their order: one already waiting on delivery, check or recheck, and when none -// waits, a pull of the delivery worker that waits for one. +// next is one ask from the workers, the kinds taking turns: the pull starts at the kind after the one the last ask came +// from and takes one already waiting on delivery, check or recheck, and when none waits, a pull of the delivery worker +// that waits for one. A fixed order let 89 waiting deliveries hold every pipeline's check back (lead-47, 2026-11-15; the +// operator's yes at 18:3x): with turns, neither kind starves the other. // // A kind whose worker handed an ask this node declined is left out until that wait ended (issue 890): the pull's own // wait, no timer of its own. func (p *Puller) next(ctx context.Context, wait time.Duration) (*Pulled, error) { p.mu.Lock() - declined := p.declined + declined, start := p.declined, p.turn p.mu.Unlock() - for _, kind := range sdkstep.AskKinds { + for i := range sdkstep.AskKinds { + at := (start + i) % len(sdkstep.AskKinds) + kind := sdkstep.AskKinds[at] if declined[kind] { continue } msg, err := p.Source.Next(ctx, kind, 0) if err != nil || msg != nil { + if msg != nil { + p.mu.Lock() + p.turn = (at + 1) % len(sdkstep.AskKinds) + p.mu.Unlock() + } return withKind(msg, kind), err } } diff --git a/modules/mesh-delivery-sdk/steprun/pull_test.go b/modules/mesh-delivery-sdk/steprun/pull_test.go index 88dfb77f..94e72343 100644 --- a/modules/mesh-delivery-sdk/steprun/pull_test.go +++ b/modules/mesh-delivery-sdk/steprun/pull_test.go @@ -106,6 +106,35 @@ func TestPullTakesDeliveryBeforeCheck(t *testing.T) { } } +// The kinds take turns (lead-47, 2026-11-15): with two deliveries and a check waiting and one slot, the check runs second, +// never held back until every delivery is done. +func TestPullKindsTakeTurns(t *testing.T) { + src, st := NewFakeAsks(), newSteps() + for _, a := range []sdkstep.StepAsk{testAsk(sdkstep.AskDelivery), testAsk(sdkstep.AskDelivery), testAsk(sdkstep.AskCheck)} { + if err := src.PublishAsk(a); err != nil { + t.Fatal(err) + } + } + var mu sync.Mutex + var order []string + p := &Puller{Source: src, Language: sdkstep.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 1 }, + PullWait: 10 * time.Millisecond, Run: func(ctx context.Context, a sdkstep.StepAsk) (any, error) { + mu.Lock() + order = append(order, a.Kind) + mu.Unlock() + return map[string]string{"ran": a.ID}, nil + }} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go p.Pull(ctx) + waitFor(t, "three asks acked", func() bool { a, _, _ := src.Fates(); return len(a) == 3 }) + mu.Lock() + defer mu.Unlock() + if want := []string{sdkstep.AskDelivery, sdkstep.AskCheck, sdkstep.AskDelivery}; len(order) != 3 || order[0] != want[0] || order[1] != want[1] || order[2] != want[2] { + t.Fatalf("order = %v, want %v", order, want) + } +} + // 1887/A2a, 1887/AC-pull: given capacity 2 and three asks waiting, two run and the third waits, and it starts when // one ends, with no timer. Issue 1806's follow-up: the 2 is the CPU's, a configured 3 bounded by a 12-core slice at 6 // cores a step, said once in capacity.; and issue 1890: a run's phase is written to its step key and logged with