Let go of the other queues while an ask runs, and read the let-through race right
A message that reaches a builder's subscription outside a fetch was held unheartbeated for the whole run of another ask, behind which a delivery waited (issue 1747). The queues the running ask did not come from are let go of and bound again after it.
This commit is contained in:
@@ -307,6 +307,19 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
|
||||
}
|
||||
continue
|
||||
}
|
||||
// The seat's own queue is bound again when a running ask made the machine let go of it (quiesce): a machine that cannot
|
||||
// is not taking, which is said and tried again.
|
||||
if !m.subs[0].bound {
|
||||
if err := m.bind(m.subs[0]); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "this machine cannot bind %s again after a build, and tries again: %v\n", m.seat, err)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-time.After(pausedPoll):
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
if time.Now().After(bindAgain) {
|
||||
for _, q := range m.subs[1:] {
|
||||
if !q.bound {
|
||||
@@ -331,7 +344,7 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
|
||||
// say why (2026-10-03, the first build agents). Asked again.
|
||||
continue
|
||||
default:
|
||||
if m.subs[0].sub.IsValid() {
|
||||
if m.subs[0].sub != nil && m.subs[0].sub.IsValid() {
|
||||
// A transient fault in asking — a reconnect, a slow server — is asked past rather
|
||||
// than ending the machine; one that outlasts the ack wait redelivers nothing lost.
|
||||
time.Sleep(time.Second)
|
||||
@@ -340,6 +353,8 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
|
||||
return fmt.Errorf("the bus stopped delivering build work: %w", err)
|
||||
}
|
||||
m.handOver(ctx, msg, q, do)
|
||||
// And the queues it let go of are bound again at once, not after the half minute.
|
||||
bindAgain = time.Time{}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -407,11 +422,34 @@ func (m *natsMachine) handOver(ctx context.Context, msg *nats.Msg, q *takeQueue,
|
||||
// And a check says it is running, to the other builders (ADR 0406 §3): its key put now and at each heartbeat,
|
||||
// deleted when it ends, whatever the outcome.
|
||||
stopRunning := keepRunning(ctx, m.js.Conn(), request, m.on, heartbeatWhileWorking)
|
||||
// **And holds nothing else while it runs** (issue 1747): see quiesce.
|
||||
m.quiesce(q)
|
||||
do(ctx, build)
|
||||
stopRunning()
|
||||
close(working)
|
||||
}
|
||||
|
||||
// quiesce lets go of every queue but the one the running ask came from, for as long as it runs. **A builder never holds a
|
||||
// message it does not start** (issue 1747). A message the server sends after a fetch gave up, within the few milliseconds its
|
||||
// pull request still lived, lands in that queue's subscription and stays there until the next fetch of it: held, never
|
||||
// acknowledged as in progress, for as long as the builder runs its current ask, five or ten minutes for a check, while the
|
||||
// holders that could build it are all busy and a delivery waits behind it. The legacy pull subscription cannot be read
|
||||
// without sending a new pull request, which could take one more, so it cannot be searched for such a message. It is dropped
|
||||
// instead: the subscription is closed, a message in it or on its way is lost to this builder and the server hands it out again
|
||||
// when its worker's wait for an acknowledgement ends (a minute), to any holder that is free, and the queue is bound again
|
||||
// before the next fetch (Take). The queue the running ask came from keeps its subscription, which its acknowledgements use.
|
||||
func (m *natsMachine) quiesce(keep *takeQueue) {
|
||||
for _, q := range m.subs {
|
||||
if q == keep || !q.bound {
|
||||
continue
|
||||
}
|
||||
if q.sub != nil {
|
||||
_ = q.sub.Unsubscribe()
|
||||
}
|
||||
q.sub, q.pull, q.bound = nil, nil, false
|
||||
}
|
||||
}
|
||||
|
||||
// runningElsewhere says why an ask need not run here, or nothing: the bucket of the checks running says another builder
|
||||
// runs this very ask (its id, its head and its trunk commit). A key that cannot be read runs the check.
|
||||
func (m *natsMachine) runningElsewhere(ctx context.Context, r BuildRequest) string {
|
||||
@@ -499,6 +537,10 @@ func (m *natsMachine) next(ctx context.Context) (*nats.Msg, *takeQueue, error) {
|
||||
|
||||
// nextBuild waits for a second at most for a delivery build, the seat's own queue.
|
||||
func (m *natsMachine) nextBuild(ctx context.Context) (*nats.Msg, *takeQueue, error) {
|
||||
if !m.subs[0].bound {
|
||||
// Let go of while an ask ran and not bound again yet (Take binds it before it asks): nothing to take from.
|
||||
return nil, nil, nil
|
||||
}
|
||||
waiting, end := context.WithTimeout(ctx, takeIdleWait)
|
||||
defer end()
|
||||
fetched, err := m.subs[0].pull.Fetch(1, nats.Context(waiting))
|
||||
|
||||
Reference in New Issue
Block a user