Merge pull request 'pipeline runner: a step-log read never waits for a batch it cannot fill' (#241) from fix/47-G-runner-ends into main
This commit was merged in pull request #241.
This commit is contained in:
@@ -62,6 +62,14 @@ func (l stepLog) Read(from uint64, limit int) ([]pipeline.LoggedEntry, error) {
|
||||
if info.State.Msgs == 0 || from > info.State.LastSeq {
|
||||
return nil, nil
|
||||
}
|
||||
// The batch asked is what the log holds from there, never more: a pull for 20 entries where fewer exist waits its whole
|
||||
// MaxWait (2 s) for the rest, and the ageing reads three retentions at every step's end.
|
||||
if from < info.State.FirstSeq {
|
||||
from = info.State.FirstSeq
|
||||
}
|
||||
if held := info.State.LastSeq - from + 1; held < uint64(limit) {
|
||||
limit = int(held)
|
||||
}
|
||||
cons, err := st.OrderedConsumer(l.ctx, jetstream.OrderedConsumerConfig{DeliverPolicy: jetstream.DeliverByStartSequencePolicy, OptStartSeq: from})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/testbus"
|
||||
)
|
||||
|
||||
func TestStepLogReadDoesNotWaitForABatchItCannotFill(t *testing.T) {
|
||||
nc, err := nats.Connect(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
js, _ := jetstream.New(nc)
|
||||
ctx := context.Background()
|
||||
const module = "m"
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(module, seat.StateStepLog), Subjects: []string{buscompose.LogSubject(module, seat.StateStepLog) + ".>"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
l := stepLog{ctx: ctx, js: js, module: module}
|
||||
for i := 0; i < 2000; i++ {
|
||||
if err := l.Append(seat.LogEntry{Pipeline: fmt.Sprintf("main.%d.x", i), Step: "lint", What: seat.LogEntryWhatEnded, At: time.Now()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// A read of the entries the log holds from a sequence returns at once, whether it holds more than the batch asked, fewer or
|
||||
// none: the ageing reads it three times at every step's end, and a pull for a batch the log cannot fill waited 2 s each.
|
||||
for _, c := range []struct {
|
||||
from uint64
|
||||
want int
|
||||
}{{1, 20}, {1990, 11}, {2001, 0}, {5000, 0}} {
|
||||
t0 := time.Now()
|
||||
got, err := l.Read(c.from, 20)
|
||||
if err != nil || len(got) != c.want || time.Since(t0) > time.Second {
|
||||
t.Errorf("Read(%d, 20) = %d entries (%v) in %v: want %d entries within a second", c.from, len(got), err, time.Since(t0), c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user