Files
mesh-sdk/go/process/queue_integration_test.go
T
jschoubben c1c369f50c H-A-196: a process pulls a durable consumer once
A second pull loop on a durable the same process already pulls takes every other
event from the first, and nothing said so: the tool runner's two loops on the
indexer bundle's durable left the catalog seat's revision verb stale twice today.
The SDK now refuses it at bind time, naming the durable and both binders:
PullOnce claims a durable (by stream, name and the time the bus made it), and
HearEvents and Conn.Consume claim through it, releasing on stop, drain or the
bus ending the loop.

Builds: design 47 H H-A-196
2026-11-20 22:05:14 +01:00

233 lines
7.5 KiB
Go

//go:build integration
package process
import (
"context"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go/jetstream"
)
// mailSendsApprovedSubject is where the runtime publishes a work-queue message on mesh-mail's own stream
// (mesh-controller internal/broker/modstreams.go ModuleStreamSubject, mail-sends' one event).
const mailSendsApprovedSubject = "mesh.stream.mesh-mail.event.mail.send.approved"
// makeQueue sets up MAIL_SENDS as a work-queue stream and binds the one durable consumer a process's
// bus.queues names it (QueueConsumerName), both as the admin, mirroring what the controller composes
// (mesh-controller internal/broker/modstreams.go QueueConsumer), named <node>_<module>_<process id>.
// ackWait is the controller's 30s in most tests, shortened in the Term test so waiting past it (to prove
// no redelivery) does not make that test slow.
func makeQueue(t *testing.T, url string, account Account, ackWait time.Duration) {
t.Helper()
_, adminJetStream := adminJS(t, url)
if _, err := adminJetStream.CreateStream(ctxFor(t), jetstream.StreamConfig{
Name: "MAIL_SENDS", Subjects: []string{mailSendsApprovedSubject}, Retention: jetstream.WorkQueuePolicy,
Discard: jetstream.DiscardNew,
}); err != nil {
t.Fatalf("the admin could not make the work-queue stream: %v", err)
}
if _, err := adminJetStream.CreateConsumer(ctxFor(t), "MAIL_SENDS", jetstream.ConsumerConfig{
Durable: account.QueueConsumerName(), AckPolicy: jetstream.AckExplicitPolicy, AckWait: ackWait,
MaxDeliver: 5,
}); err != nil {
t.Fatalf("the admin could not make the queue consumer: %v", err)
}
}
func publishApproved(t *testing.T, url string, body string) {
t.Helper()
nc, _ := adminJS(t, url)
if err := nc.Publish(mailSendsApprovedSubject, []byte(body)); err != nil {
t.Fatal(err)
}
if err := nc.FlushTimeout(3 * time.Second); err != nil {
t.Fatal(err)
}
}
// recv reads one value off ch, failing the test rather than hanging forever if nothing arrives within
// timeout.
func recv[T any](t *testing.T, ch <-chan T, timeout time.Duration, what string) T {
t.Helper()
select {
case v := <-ch:
return v
case <-time.After(timeout):
t.Fatalf("%s: nothing arrived within %s", what, timeout)
var zero T
return zero
}
}
// streamMsgs is how many messages MAIL_SENDS still holds.
func streamMsgs(t *testing.T, url string) uint64 {
t.Helper()
_, adminJetStream := adminJS(t, url)
stream, err := adminJetStream.Stream(ctxFor(t), "MAIL_SENDS")
if err != nil {
t.Fatal(err)
}
info, err := stream.Info(ctxFor(t))
if err != nil {
t.Fatal(err)
}
return info.State.Msgs
}
// 1190/AC1: given a work-queue stream with one durable pull consumer and a process connection, when a
// message is consumed and acknowledged, then it is gone from the stream.
func TestConsumeAckRemovesTheMessageFromTheStream(t *testing.T) {
url, account := grantedSendProcess(t)
makeQueue(t, url, account, 30*time.Second)
publishApproved(t, url, `{"send":"s1"}`)
conn, err := Connect(account)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
got, second := make(chan *Message, 1), make(chan error, 1)
go func() {
_ = conn.Consume(ctx, "MAIL_SENDS", func(m *Message) {
// A second pull loop on the durable this process pulls is refused at bind time (design 47 H, H-A-196).
short, stop := context.WithTimeout(context.Background(), time.Second)
second <- conn.Consume(short, "MAIL_SENDS", func(*Message) {})
stop()
got <- m
cancel()
})
}()
msg := recv(t, got, 8*time.Second, "the message")
<-ctx.Done()
if err := <-second; err == nil || !strings.Contains(err.Error(), "pulled in this process already") ||
!strings.Contains(err.Error(), account.QueueConsumerName()) {
t.Fatalf("a second Consume of the queue the process pulls was not refused naming its durable: %v", err)
}
if err := msg.Ack(); err != nil {
t.Fatalf("Ack: %v", err)
}
deadline := time.Now().Add(5 * time.Second)
for {
if n := streamMsgs(t, url); n == 0 {
return
} else if time.Now().After(deadline) {
t.Fatalf("the stream still holds %d messages after the ack", n)
}
time.Sleep(100 * time.Millisecond)
}
}
// 1190/AC2: given a consumed message, when the handler asks for it to be delivered again after a delay,
// then it comes back no sooner than the delay with its delivery count raised by one.
func TestConsumeNakWithDelayRedeliversAfterTheDelayWithARaisedCount(t *testing.T) {
url, account := grantedSendProcess(t)
makeQueue(t, url, account, 30*time.Second)
publishApproved(t, url, `{"send":"s2"}`)
conn, err := Connect(account)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
const delay = 2 * time.Second
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
type delivery struct {
at time.Time
delivered uint64
}
deliveries := make(chan delivery, 2)
go func() {
_ = conn.Consume(ctx, "MAIL_SENDS", func(m *Message) {
deliveries <- delivery{time.Now(), m.Delivered}
if m.Delivered == 1 {
_ = m.NakWithDelay(delay)
} else {
_ = m.Ack()
cancel()
}
})
}()
first := recv(t, deliveries, 8*time.Second, "the first delivery")
if first.delivered != 1 {
t.Fatalf("first delivery count: %d, want 1", first.delivered)
}
second := recv(t, deliveries, delay+8*time.Second, "the redelivery")
if second.delivered != 2 {
t.Fatalf("second delivery count: %d, want 2", second.delivered)
}
if elapsed := second.at.Sub(first.at); elapsed < delay {
t.Fatalf("redelivered after %s, want no sooner than %s", elapsed, delay)
}
}
// 1190/AC3: given a consumed message, when the handler terminates it, then it is never delivered again.
// A short AckWait (well under the 4s this then waits past) is what would otherwise make this pass
// whether or not Term did anything at all: without it, the message simply sits unredelivered until the
// controller's ordinary 30s AckWait, and the test would see the same "nothing arrived" result either way.
func TestConsumeTermNeverDeliversTheMessageAgain(t *testing.T) {
url, account := grantedSendProcess(t)
const ackWait = time.Second
makeQueue(t, url, account, ackWait)
publishApproved(t, url, `{"send":"s3"}`)
conn, err := Connect(account)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
deliveries := make(chan *Message, 4)
go func() {
_ = conn.Consume(ctx, "MAIL_SENDS", func(m *Message) {
deliveries <- m
_ = m.Term()
})
}()
first := recv(t, deliveries, 8*time.Second, "the message")
if first.Delivered != 1 {
t.Fatalf("first delivery count: %d, want 1", first.Delivered)
}
// Term removes it from the work-queue stream and leaves the consumer with nothing pending — not just
// "has not come back yet", which a message merely sitting past an un-acted AckWait would also show.
deadline := time.Now().Add(5 * time.Second)
for {
if n := streamMsgs(t, url); n == 0 {
break
} else if time.Now().After(deadline) {
t.Fatalf("the stream still holds %d messages after Term", n)
}
time.Sleep(100 * time.Millisecond)
}
_, adminJetStream := adminJS(t, url)
cons, err := adminJetStream.Consumer(ctxFor(t), "MAIL_SENDS", account.QueueConsumerName())
if err != nil {
t.Fatal(err)
}
info, err := cons.Info(ctxFor(t))
if err != nil {
t.Fatal(err)
}
if info.NumAckPending != 0 {
t.Fatalf("the consumer still has %d acks pending after Term", info.NumAckPending)
}
select {
case again := <-deliveries:
t.Fatalf("a terminated message was delivered again: %+v", again)
case <-time.After(4 * ackWait):
}
}