diff --git a/go/process/event_hub_test.go b/go/process/event_hub_test.go index c6ac390d..8920aee2 100644 --- a/go/process/event_hub_test.go +++ b/go/process/event_hub_test.go @@ -4,6 +4,7 @@ import ( "context" "strings" "testing" + "time" "github.com/nats-io/nats.go/jetstream" ) @@ -66,3 +67,83 @@ func TestTheHubMatchesSubjectsAndRefusesOverlappingPatterns(t *testing.T) { t.Errorf("the durable on a stream kept with an age starts from its first message, got %v", cfg.DeliverPolicy) } } + +type pullConsumer struct { + jetstream.Consumer + info *jetstream.ConsumerInfo + handle jetstream.MessageHandler + options []jetstream.PullConsumeOpt +} + +func (c *pullConsumer) CachedInfo() *jetstream.ConsumerInfo { return c.info } +func (c *pullConsumer) Consume(handle jetstream.MessageHandler, opts ...jetstream.PullConsumeOpt) (jetstream.ConsumeContext, error) { + c.handle, c.options = handle, opts + return &pullContext{closed: make(chan struct{})}, nil +} + +type pullContext struct { + jetstream.ConsumeContext + closed chan struct{} +} + +func (p *pullContext) Stop() { close(p.closed) } +func (p *pullContext) Closed() <-chan struct{} { return p.closed } + +type slowMsg struct { + jetstream.Msg + said chan struct{} +} + +func (m *slowMsg) InProgress() error { m.said <- struct{}{}; return nil } + +// H-A-281: the hub pulls at most twenty messages per pull, and says in progress for a message whose handler runs past a +// third of the consumer's ack wait, at each third until it returns; a handler that returns first says nothing. +func TestTheHubPullsSmallBatchesAndKeepsSlowHandlersAlive(t *testing.T) { + cons := &pullConsumer{info: &jetstream.ConsumerInfo{Config: jetstream.ConsumerConfig{AckWait: 30 * time.Second}}} + ticks := make(chan time.Time) + var every time.Duration + stopped := 0 + tick := func(d time.Duration) (<-chan time.Time, func()) { every = d; return ticks, func() { stopped++ } } + var handled func(jetstream.Msg) + cc, err := hearEvents(cons, "the test", func(m jetstream.Msg) { handled(m) }, t.Logf, tick) + if err != nil { + t.Fatal(err) + } + defer cc.Stop() + pulls := 0 + for _, o := range cons.options { + if n, ok := o.(jetstream.PullMaxMessages); ok { + pulls = int(n) + } + } + if pulls != 20 { + t.Errorf("the hub pulls %d messages per pull, want 20 (0: the bus client's default of 500)", pulls) + } + slow := &slowMsg{said: make(chan struct{}, 4)} + handled = func(jetstream.Msg) { + for range 2 { // two thirds of the ack wait pass while the handler runs + select { + case ticks <- time.Time{}: + case <-time.After(5 * time.Second): + t.Error("no ticker runs while the handler runs") + return + } + select { + case <-slow.said: + case <-time.After(5 * time.Second): + t.Error("a handler running past a third of the ack wait said no in progress") + return + } + } + } + cons.handle(slow) + if every != 10*time.Second || stopped != 1 { + t.Errorf("in progress every %v (want 10s, a third of the ack wait), ticker stopped %d times (want 1)", every, stopped) + } + fast := &slowMsg{said: make(chan struct{}, 1)} + handled = func(jetstream.Msg) {} + cons.handle(fast) + if len(fast.said) != 0 { + t.Error("a handler that returned before a third of the ack wait said in progress") + } +} diff --git a/go/process/module.go b/go/process/module.go index 57a337c8..d740effc 100644 --- a/go/process/module.go +++ b/go/process/module.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "strings" + "time" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" @@ -106,8 +107,58 @@ func (a Account) namedEventConsumerName(suffix string) string { // HearEvents pulls from cons, handing each message to handle, and says every error the bus reports while // hearing through logf ("hearing the bus: ..."). The caller stops the returned context. A durable this process pulls // already is refused (ConsumeOnce, design 47 H, H-A-196). +// +// It pulls at most hubPullMessages messages per pull, and says in progress to the bus for a message whose handler runs +// past a third of the consumer's ack wait, again at each further third until the handler returns (design 47 H, H-A-281): +// handlers run one by one, so a large pull's buffered messages would outlive the ack wait and be given again. The ack +// wait is read once, from the consumer's information as the bind read it. func HearEvents(cons jetstream.Consumer, handle func(jetstream.Msg), logf func(string, ...any)) (jetstream.ConsumeContext, error) { - return ConsumeOnce(cons, callerOutside(), handle, jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { - logf("hearing the bus: %v", err) - })) + return hearEvents(cons, callerOutside(), handle, logf, realTicker) +} + +// hubPullMessages is how many messages the event hub pulls at most per pull (design 47 H, H-A-281). +const hubPullMessages = 20 + +// ticker answers a channel ticking every d and its stop. +type ticker func(d time.Duration) (<-chan time.Time, func()) + +func realTicker(d time.Duration) (<-chan time.Time, func()) { + t := time.NewTicker(d) + return t.C, t.Stop +} + +func hearEvents(cons jetstream.Consumer, binder string, handle func(jetstream.Msg), logf func(string, ...any), tick ticker) (jetstream.ConsumeContext, error) { + var ackWait time.Duration + if info := cons.CachedInfo(); info != nil { + ackWait = info.Config.AckWait + } + return ConsumeOnce(cons, binder, keptInProgress(handle, ackWait/3, tick), jetstream.PullMaxMessages(hubPullMessages), + jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { + logf("hearing the bus: %v", err) + })) +} + +// keptInProgress is handle that says in progress for its message at every tick of every while the handler runs, the +// ticker stopped at the handler's return (design 47 H, H-A-281); with no every it is handle as it is. +func keptInProgress(handle func(jetstream.Msg), every time.Duration, tick ticker) jetstream.MessageHandler { + if every <= 0 { + return handle + } + return func(m jetstream.Msg) { + ticks, stop := tick(every) + done, ended := make(chan struct{}), make(chan struct{}) + go func() { + defer close(ended) + for { + select { + case <-done: + return + case <-ticks: + _ = m.InProgress() + } + } + }() + defer func() { close(done); stop(); <-ended }() + handle(m) + } }