diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go index c31c62c8d..71cf6a75b 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go @@ -643,38 +643,28 @@ func (pq *publishQueue) pull(ctx context.Context, holder string) error { if err != nil { return err } - chans := map[string]chan jetstream.Msg{} - var live jetstream.Consumer - for _, class := range []string{versions.PublishClassStart, versions.PublishClassLive} { // the again first: the live is first by the dispatch, not by luck + cons := map[string]jetstream.Consumer{} + for _, class := range []string{versions.PublishClassStart, versions.PublishClassLive} { c, err := pq.js.CreateOrUpdateConsumer(ctx, b.Stream, jetstream.ConsumerConfig{Durable: publishesConsumer(holder, class), FilterSubject: b.Subject + ".publish." + versions.EscapeToken(holder) + "." + class + ".>", AckPolicy: jetstream.AckExplicitPolicy, AckWait: pq.ackWait, MaxAckPending: publishesMaxAckPending, MaxDeliver: publishesMaxDeliver}) if err != nil { return fmt.Errorf("the consumer %s: %w", publishesConsumer(holder, class), err) } - ch := make(chan jetstream.Msg, publishesMaxAckPending) - cc, err := c.Consume(func(m jetstream.Msg) { ch <- m }, jetstream.PullMaxMessages(publishesMaxAckPending)) - if err != nil { - return fmt.Errorf("the consumer %s is not pulled: %w", publishesConsumer(holder, class), err) - } - go func() { <-ctx.Done(); cc.Stop() }() - chans[class] = ch - if class == versions.PublishClassLive { - live = c - } - } - liveWaits := func() bool { - info, err := live.Info(ctx) - return err == nil && info.NumPending > 0 + cons[class] = c } pq.logf("pulling the publishes of %s, the live first", holder) - go pq.dispatch(ctx, chans[versions.PublishClassLive], chans[versions.PublishClassStart], liveWaits) + go pq.dispatch(ctx, holder, cons[versions.PublishClassLive], cons[versions.PublishClassStart]) return nil } -// dispatch runs the items of one holder, at most four at once, every live item before an again one: an again item pulled while -// a live one waits on the queue (liveWaits) runs after it. -func (pq *publishQueue) dispatch(ctx context.Context, live, again <-chan jetstream.Msg, liveWaits func() bool) { +// dispatch runs the items of one holder, at most four at once, every live item before an again one (design 47 H2 ยง793, +// H-A-473): it pulls the live consumer itself, a fetch of up to its free slots that does not wait, and pulls the again consumer +// only when that fetch returns nothing; an item is received the moment its fetch returns it, so no live item is delivered and +// not yet received while an again one is chosen. When neither holds an item, a pull of one item waits on each, and what it +// receives is run, the live first. A pull the bus refuses ends the dispatch, said; the holder is pulled again when an item of +// it is put. +func (pq *publishQueue) dispatch(ctx context.Context, holder string, live, again jetstream.Consumer) { sem := make(chan struct{}, versions.PublishSendsAtOnce) run := func(m jetstream.Msg) { go func() { @@ -682,48 +672,142 @@ func (pq *publishQueue) dispatch(ctx context.Context, live, again <-chan jetstre pq.take(ctx, m) }() } - var spare jetstream.Msg // an again item pulled while a live one waited + failed := func(err error) { + pq.logf("the publishes of %s are not pulled: %v; they are pulled when an item of it is put", holder, err) + pq.mu.Lock() + delete(pq.pulling, holder) + pq.mu.Unlock() + } + refused := func(err error) bool { + return err != nil && !errors.Is(err, jetstream.ErrNoMessages) && !errors.Is(err, nats.ErrTimeout) && ctx.Err() == nil + } + now := func(c jetstream.Consumer, n int) ([]jetstream.Msg, error) { + b, err := c.FetchNoWait(n) + if err != nil { + return nil, err + } + var got []jetstream.Msg + for m := range b.Messages() { + got = append(got, m) + } + return got, b.Error() + } + type waited struct { + m jetstream.Msg + err error + } + liveDone, againDone := make(chan waited, 1), make(chan waited, 1) + wait := func(c jetstream.Consumer, done chan<- waited) { + go func() { + var w waited + b, err := c.Fetch(1) + if err == nil { + for m := range b.Messages() { + w.m = m + } + err = b.Error() + } + w.err = err + done <- w + }() + } + var heldLive, heldAgain jetstream.Msg // an item a waiting pull received + var liveWaiting, againWaiting bool + heard := func(w waited, held *jetstream.Msg, waiting *bool) bool { + *waiting = false + if refused(w.err) { + failed(w.err) + return false + } + if w.m != nil { + *held = w.m + } + return true + } for { select { case sem <- struct{}{}: case <-ctx.Done(): return } - select { - case m := <-live: - run(m) - continue - default: - } - m := spare - spare = nil - if m == nil { + n := 1 + slots: + for n < cap(sem) { select { - case lm := <-live: - run(lm) - continue - case m = <-again: - case <-ctx.Done(): - return + case sem <- struct{}{}: + n++ + default: + break slots } } - if liveWaits() { - spare = m - select { - case lm := <-live: - run(lm) - case <-ctx.Done(): + select { + case w := <-liveDone: + if !heard(w, &heldLive, &liveWaiting) { return } - continue + default: } select { - case lm := <-live: // came while the queue was asked - spare = m - run(lm) + case w := <-againDone: + if !heard(w, &heldAgain, &againWaiting) { + return + } default: + } + var got []jetstream.Msg + switch { + case heldLive != nil: + got, heldLive = []jetstream.Msg{heldLive}, nil + case !liveWaiting: + ms, err := now(live, n) + if refused(err) { + failed(err) + return + } + got = ms + } + if len(got) == 0 { + switch { + case heldAgain != nil: + got, heldAgain = []jetstream.Msg{heldAgain}, nil + case !againWaiting: + ms, err := now(again, n) + if refused(err) { + failed(err) + return + } + got = ms + } + } + for _, m := range got { run(m) } + for range n - len(got) { + <-sem + } + if len(got) > 0 { + continue + } + if !liveWaiting { + liveWaiting = true + wait(live, liveDone) + } + if !againWaiting { + againWaiting = true + wait(again, againDone) + } + select { + case w := <-liveDone: + if !heard(w, &heldLive, &liveWaiting) { + return + } + case w := <-againDone: + if !heard(w, &heldAgain, &againWaiting) { + return + } + case <-ctx.Done(): + return + } } }