Let go of the other queues while an ask runs, and read the let-through race right

A message that reaches a builder's subscription outside a fetch was held unheartbeated for
the whole run of another ask, behind which a delivery waited (issue 1747). The queues the
running ask did not come from are let go of and bound again after it.
This commit is contained in:
2026-10-21 19:56:23 +02:00
parent a14dc04b7b
commit bb4d2e1da4
3 changed files with 201 additions and 10 deletions
+43 -1
View File
@@ -307,6 +307,19 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
}
continue
}
// The seat's own queue is bound again when a running ask made the machine let go of it (quiesce): a machine that cannot
// is not taking, which is said and tried again.
if !m.subs[0].bound {
if err := m.bind(m.subs[0]); err != nil {
fmt.Fprintf(os.Stderr, "this machine cannot bind %s again after a build, and tries again: %v\n", m.seat, err)
select {
case <-ctx.Done():
return nil
case <-time.After(pausedPoll):
}
continue
}
}
if time.Now().After(bindAgain) {
for _, q := range m.subs[1:] {
if !q.bound {
@@ -331,7 +344,7 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
// say why (2026-10-03, the first build agents). Asked again.
continue
default:
if m.subs[0].sub.IsValid() {
if m.subs[0].sub != nil && m.subs[0].sub.IsValid() {
// A transient fault in asking — a reconnect, a slow server — is asked past rather
// than ending the machine; one that outlasts the ack wait redelivers nothing lost.
time.Sleep(time.Second)
@@ -340,6 +353,8 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
return fmt.Errorf("the bus stopped delivering build work: %w", err)
}
m.handOver(ctx, msg, q, do)
// And the queues it let go of are bound again at once, not after the half minute.
bindAgain = time.Time{}
}
}
@@ -407,11 +422,34 @@ func (m *natsMachine) handOver(ctx context.Context, msg *nats.Msg, q *takeQueue,
// And a check says it is running, to the other builders (ADR 0406 §3): its key put now and at each heartbeat,
// deleted when it ends, whatever the outcome.
stopRunning := keepRunning(ctx, m.js.Conn(), request, m.on, heartbeatWhileWorking)
// **And holds nothing else while it runs** (issue 1747): see quiesce.
m.quiesce(q)
do(ctx, build)
stopRunning()
close(working)
}
// quiesce lets go of every queue but the one the running ask came from, for as long as it runs. **A builder never holds a
// message it does not start** (issue 1747). A message the server sends after a fetch gave up, within the few milliseconds its
// pull request still lived, lands in that queue's subscription and stays there until the next fetch of it: held, never
// acknowledged as in progress, for as long as the builder runs its current ask, five or ten minutes for a check, while the
// holders that could build it are all busy and a delivery waits behind it. The legacy pull subscription cannot be read
// without sending a new pull request, which could take one more, so it cannot be searched for such a message. It is dropped
// instead: the subscription is closed, a message in it or on its way is lost to this builder and the server hands it out again
// when its worker's wait for an acknowledgement ends (a minute), to any holder that is free, and the queue is bound again
// before the next fetch (Take). The queue the running ask came from keeps its subscription, which its acknowledgements use.
func (m *natsMachine) quiesce(keep *takeQueue) {
for _, q := range m.subs {
if q == keep || !q.bound {
continue
}
if q.sub != nil {
_ = q.sub.Unsubscribe()
}
q.sub, q.pull, q.bound = nil, nil, false
}
}
// runningElsewhere says why an ask need not run here, or nothing: the bucket of the checks running says another builder
// runs this very ask (its id, its head and its trunk commit). A key that cannot be read runs the check.
func (m *natsMachine) runningElsewhere(ctx context.Context, r BuildRequest) string {
@@ -499,6 +537,10 @@ func (m *natsMachine) next(ctx context.Context) (*nats.Msg, *takeQueue, error) {
// nextBuild waits for a second at most for a delivery build, the seat's own queue.
func (m *natsMachine) nextBuild(ctx context.Context) (*nats.Msg, *takeQueue, error) {
if !m.subs[0].bound {
// Let go of while an ask ran and not bound again yet (Take binds it before it asks): nothing to take from.
return nil, nil, nil
}
waiting, end := context.WithTimeout(ctx, takeIdleWait)
defer end()
fetched, err := m.subs[0].pull.Fetch(1, nats.Context(waiting))