steprun: the build seats' queue kinds take turns
A build module pulled a waiting delivery before any check, so 89 queued deliveries held every pipeline's check back on shanks for an hour and more. The pull now starts at the kind after the one its last ask came from; neither kind starves the other (the operator's yes, 2026-11-15).
This commit is contained in:
@@ -169,6 +169,8 @@ type Puller struct {
|
||||
// declined are the kinds whose worker handed an ask this node declined, left out of the pulls without a wait until
|
||||
// the next pull's wait ends.
|
||||
declined map[string]bool
|
||||
// turn is the kind the next pull asks first (next): the kinds take turns, so a queue that never empties holds back none.
|
||||
turn int
|
||||
}
|
||||
|
||||
func (p *Puller) init() {
|
||||
@@ -519,21 +521,30 @@ func (p *Puller) Pull(ctx context.Context) error {
|
||||
}
|
||||
}
|
||||
|
||||
// next is one ask from the workers in their order: one already waiting on delivery, check or recheck, and when none
|
||||
// waits, a pull of the delivery worker that waits for one.
|
||||
// next is one ask from the workers, the kinds taking turns: the pull starts at the kind after the one the last ask came
|
||||
// from and takes one already waiting on delivery, check or recheck, and when none waits, a pull of the delivery worker
|
||||
// that waits for one. A fixed order let 89 waiting deliveries hold every pipeline's check back (lead-47, 2026-11-15; the
|
||||
// operator's yes at 18:3x): with turns, neither kind starves the other.
|
||||
//
|
||||
// A kind whose worker handed an ask this node declined is left out until that wait ended (issue 890): the pull's own
|
||||
// wait, no timer of its own.
|
||||
func (p *Puller) next(ctx context.Context, wait time.Duration) (*Pulled, error) {
|
||||
p.mu.Lock()
|
||||
declined := p.declined
|
||||
declined, start := p.declined, p.turn
|
||||
p.mu.Unlock()
|
||||
for _, kind := range sdkstep.AskKinds {
|
||||
for i := range sdkstep.AskKinds {
|
||||
at := (start + i) % len(sdkstep.AskKinds)
|
||||
kind := sdkstep.AskKinds[at]
|
||||
if declined[kind] {
|
||||
continue
|
||||
}
|
||||
msg, err := p.Source.Next(ctx, kind, 0)
|
||||
if err != nil || msg != nil {
|
||||
if msg != nil {
|
||||
p.mu.Lock()
|
||||
p.turn = (at + 1) % len(sdkstep.AskKinds)
|
||||
p.mu.Unlock()
|
||||
}
|
||||
return withKind(msg, kind), err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -106,6 +106,35 @@ func TestPullTakesDeliveryBeforeCheck(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The kinds take turns (lead-47, 2026-11-15): with two deliveries and a check waiting and one slot, the check runs second,
|
||||
// never held back until every delivery is done.
|
||||
func TestPullKindsTakeTurns(t *testing.T) {
|
||||
src, st := NewFakeAsks(), newSteps()
|
||||
for _, a := range []sdkstep.StepAsk{testAsk(sdkstep.AskDelivery), testAsk(sdkstep.AskDelivery), testAsk(sdkstep.AskCheck)} {
|
||||
if err := src.PublishAsk(a); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var mu sync.Mutex
|
||||
var order []string
|
||||
p := &Puller{Source: src, Language: sdkstep.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 1 },
|
||||
PullWait: 10 * time.Millisecond, Run: func(ctx context.Context, a sdkstep.StepAsk) (any, error) {
|
||||
mu.Lock()
|
||||
order = append(order, a.Kind)
|
||||
mu.Unlock()
|
||||
return map[string]string{"ran": a.ID}, nil
|
||||
}}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go p.Pull(ctx)
|
||||
waitFor(t, "three asks acked", func() bool { a, _, _ := src.Fates(); return len(a) == 3 })
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if want := []string{sdkstep.AskDelivery, sdkstep.AskCheck, sdkstep.AskDelivery}; len(order) != 3 || order[0] != want[0] || order[1] != want[1] || order[2] != want[2] {
|
||||
t.Fatalf("order = %v, want %v", order, want)
|
||||
}
|
||||
}
|
||||
|
||||
// 1887/A2a, 1887/AC-pull: given capacity 2 and three asks waiting, two run and the third waits, and it starts when
|
||||
// one ends, with no timer. Issue 1806's follow-up: the 2 is the CPU's, a configured 3 bounded by a 12-core slice at 6
|
||||
// cores a step, said once in capacity.<node>; and issue 1890: a run's phase is written to its step key and logged with
|
||||
|
||||
Reference in New Issue
Block a user