pipeline runner: a step-log read never waits for a batch it cannot fill #241

Merged
mesh-tools merged 1 commits from fix/47-G-runner-ends into main 2026-11-17 09:40:11 +00:00
2 changed files with 56 additions and 0 deletions
@@ -62,6 +62,14 @@ func (l stepLog) Read(from uint64, limit int) ([]pipeline.LoggedEntry, error) {
if info.State.Msgs == 0 || from > info.State.LastSeq {
return nil, nil
}
// The batch asked is what the log holds from there, never more: a pull for 20 entries where fewer exist waits its whole
// MaxWait (2 s) for the rest, and the ageing reads three retentions at every step's end.
if from < info.State.FirstSeq {
from = info.State.FirstSeq
}
if held := info.State.LastSeq - from + 1; held < uint64(limit) {
limit = int(held)
}
cons, err := st.OrderedConsumer(l.ctx, jetstream.OrderedConsumerConfig{DeliverPolicy: jetstream.DeliverByStartSequencePolicy, OptStartSeq: from})
if err != nil {
return nil, err
@@ -0,0 +1,48 @@
//go:build integration
package main
import (
"context"
"fmt"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/testbus"
)
func TestStepLogReadDoesNotWaitForABatchItCannotFill(t *testing.T) {
nc, err := nats.Connect(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
js, _ := jetstream.New(nc)
ctx := context.Background()
const module = "m"
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(module, seat.StateStepLog), Subjects: []string{buscompose.LogSubject(module, seat.StateStepLog) + ".>"}}); err != nil {
t.Fatal(err)
}
l := stepLog{ctx: ctx, js: js, module: module}
for i := 0; i < 2000; i++ {
if err := l.Append(seat.LogEntry{Pipeline: fmt.Sprintf("main.%d.x", i), Step: "lint", What: seat.LogEntryWhatEnded, At: time.Now()}); err != nil {
t.Fatal(err)
}
}
// A read of the entries the log holds from a sequence returns at once, whether it holds more than the batch asked, fewer or
// none: the ageing reads it three times at every step's end, and a pull for a batch the log cannot fill waited 2 s each.
for _, c := range []struct {
from uint64
want int
}{{1, 20}, {1990, 11}, {2001, 0}, {5000, 0}} {
t0 := time.Now()
got, err := l.Read(c.from, 20)
if err != nil || len(got) != c.want || time.Since(t0) > time.Second {
t.Errorf("Read(%d, 20) = %d entries (%v) in %v: want %d entries within a second", c.from, len(got), err, time.Since(t0), c.want)
}
}
}