Pull small batches in the event hub and keep slow handlers alive (H-A-281)
The hub pulled the bus client's default 500 messages per pull and handled them one by one, so buffered messages outlived the 30 s ack wait and were given again and handled twice (met live on the builder: 37 of 38 outstanding redelivered). HearEvents, the one pull of the hub and of HearStream, now pulls at most twenty messages per pull and says in progress for a message whose handler runs past a third of the consumer's ack wait, read once from the bound consumer's information. Builds: design 47 H H-A-281
This commit is contained in:
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
@@ -66,3 +67,83 @@ func TestTheHubMatchesSubjectsAndRefusesOverlappingPatterns(t *testing.T) {
|
||||
t.Errorf("the durable on a stream kept with an age starts from its first message, got %v", cfg.DeliverPolicy)
|
||||
}
|
||||
}
|
||||
|
||||
type pullConsumer struct {
|
||||
jetstream.Consumer
|
||||
info *jetstream.ConsumerInfo
|
||||
handle jetstream.MessageHandler
|
||||
options []jetstream.PullConsumeOpt
|
||||
}
|
||||
|
||||
func (c *pullConsumer) CachedInfo() *jetstream.ConsumerInfo { return c.info }
|
||||
func (c *pullConsumer) Consume(handle jetstream.MessageHandler, opts ...jetstream.PullConsumeOpt) (jetstream.ConsumeContext, error) {
|
||||
c.handle, c.options = handle, opts
|
||||
return &pullContext{closed: make(chan struct{})}, nil
|
||||
}
|
||||
|
||||
type pullContext struct {
|
||||
jetstream.ConsumeContext
|
||||
closed chan struct{}
|
||||
}
|
||||
|
||||
func (p *pullContext) Stop() { close(p.closed) }
|
||||
func (p *pullContext) Closed() <-chan struct{} { return p.closed }
|
||||
|
||||
type slowMsg struct {
|
||||
jetstream.Msg
|
||||
said chan struct{}
|
||||
}
|
||||
|
||||
func (m *slowMsg) InProgress() error { m.said <- struct{}{}; return nil }
|
||||
|
||||
// H-A-281: the hub pulls at most twenty messages per pull, and says in progress for a message whose handler runs past a
|
||||
// third of the consumer's ack wait, at each third until it returns; a handler that returns first says nothing.
|
||||
func TestTheHubPullsSmallBatchesAndKeepsSlowHandlersAlive(t *testing.T) {
|
||||
cons := &pullConsumer{info: &jetstream.ConsumerInfo{Config: jetstream.ConsumerConfig{AckWait: 30 * time.Second}}}
|
||||
ticks := make(chan time.Time)
|
||||
var every time.Duration
|
||||
stopped := 0
|
||||
tick := func(d time.Duration) (<-chan time.Time, func()) { every = d; return ticks, func() { stopped++ } }
|
||||
var handled func(jetstream.Msg)
|
||||
cc, err := hearEvents(cons, "the test", func(m jetstream.Msg) { handled(m) }, t.Logf, tick)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer cc.Stop()
|
||||
pulls := 0
|
||||
for _, o := range cons.options {
|
||||
if n, ok := o.(jetstream.PullMaxMessages); ok {
|
||||
pulls = int(n)
|
||||
}
|
||||
}
|
||||
if pulls != 20 {
|
||||
t.Errorf("the hub pulls %d messages per pull, want 20 (0: the bus client's default of 500)", pulls)
|
||||
}
|
||||
slow := &slowMsg{said: make(chan struct{}, 4)}
|
||||
handled = func(jetstream.Msg) {
|
||||
for range 2 { // two thirds of the ack wait pass while the handler runs
|
||||
select {
|
||||
case ticks <- time.Time{}:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Error("no ticker runs while the handler runs")
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-slow.said:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Error("a handler running past a third of the ack wait said no in progress")
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
cons.handle(slow)
|
||||
if every != 10*time.Second || stopped != 1 {
|
||||
t.Errorf("in progress every %v (want 10s, a third of the ack wait), ticker stopped %d times (want 1)", every, stopped)
|
||||
}
|
||||
fast := &slowMsg{said: make(chan struct{}, 1)}
|
||||
handled = func(jetstream.Msg) {}
|
||||
cons.handle(fast)
|
||||
if len(fast.said) != 0 {
|
||||
t.Error("a handler that returned before a third of the ack wait said in progress")
|
||||
}
|
||||
}
|
||||
|
||||
+54
-3
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
@@ -106,8 +107,58 @@ func (a Account) namedEventConsumerName(suffix string) string {
|
||||
// HearEvents pulls from cons, handing each message to handle, and says every error the bus reports while
|
||||
// hearing through logf ("hearing the bus: ..."). The caller stops the returned context. A durable this process pulls
|
||||
// already is refused (ConsumeOnce, design 47 H, H-A-196).
|
||||
//
|
||||
// It pulls at most hubPullMessages messages per pull, and says in progress to the bus for a message whose handler runs
|
||||
// past a third of the consumer's ack wait, again at each further third until the handler returns (design 47 H, H-A-281):
|
||||
// handlers run one by one, so a large pull's buffered messages would outlive the ack wait and be given again. The ack
|
||||
// wait is read once, from the consumer's information as the bind read it.
|
||||
func HearEvents(cons jetstream.Consumer, handle func(jetstream.Msg), logf func(string, ...any)) (jetstream.ConsumeContext, error) {
|
||||
return ConsumeOnce(cons, callerOutside(), handle, jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) {
|
||||
logf("hearing the bus: %v", err)
|
||||
}))
|
||||
return hearEvents(cons, callerOutside(), handle, logf, realTicker)
|
||||
}
|
||||
|
||||
// hubPullMessages is how many messages the event hub pulls at most per pull (design 47 H, H-A-281).
|
||||
const hubPullMessages = 20
|
||||
|
||||
// ticker answers a channel ticking every d and its stop.
|
||||
type ticker func(d time.Duration) (<-chan time.Time, func())
|
||||
|
||||
func realTicker(d time.Duration) (<-chan time.Time, func()) {
|
||||
t := time.NewTicker(d)
|
||||
return t.C, t.Stop
|
||||
}
|
||||
|
||||
func hearEvents(cons jetstream.Consumer, binder string, handle func(jetstream.Msg), logf func(string, ...any), tick ticker) (jetstream.ConsumeContext, error) {
|
||||
var ackWait time.Duration
|
||||
if info := cons.CachedInfo(); info != nil {
|
||||
ackWait = info.Config.AckWait
|
||||
}
|
||||
return ConsumeOnce(cons, binder, keptInProgress(handle, ackWait/3, tick), jetstream.PullMaxMessages(hubPullMessages),
|
||||
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) {
|
||||
logf("hearing the bus: %v", err)
|
||||
}))
|
||||
}
|
||||
|
||||
// keptInProgress is handle that says in progress for its message at every tick of every while the handler runs, the
|
||||
// ticker stopped at the handler's return (design 47 H, H-A-281); with no every it is handle as it is.
|
||||
func keptInProgress(handle func(jetstream.Msg), every time.Duration, tick ticker) jetstream.MessageHandler {
|
||||
if every <= 0 {
|
||||
return handle
|
||||
}
|
||||
return func(m jetstream.Msg) {
|
||||
ticks, stop := tick(every)
|
||||
done, ended := make(chan struct{}), make(chan struct{})
|
||||
go func() {
|
||||
defer close(ended)
|
||||
for {
|
||||
select {
|
||||
case <-done:
|
||||
return
|
||||
case <-ticks:
|
||||
_ = m.InProgress()
|
||||
}
|
||||
}
|
||||
}()
|
||||
defer func() { close(done); stop(); <-ended }()
|
||||
handle(m)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user