From 8d93c3ff9f743f475b4520a089f56777f581fcd3 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 17 Nov 2026 10:38:13 +0100 Subject: [PATCH] Pipeline runner: the ageing reads the step log without waiting for a batch the log cannot fill MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every step's end waited 2 s per retention (6 s) in the step log's Fetch when fewer than 20 entries lay past the aged pointer, so ends were taken one in 6 s serially. Builds: design 47 G ยง330 (ageing of the runner's own data at each step's end) --- .../cmd/mesh-delivery-pipeline/aged.go | 8 ++++ .../steplog_read_integration_test.go | 48 +++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steplog_read_integration_test.go diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/aged.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/aged.go index 9a780571..25816a91 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/aged.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/aged.go @@ -62,6 +62,14 @@ func (l stepLog) Read(from uint64, limit int) ([]pipeline.LoggedEntry, error) { if info.State.Msgs == 0 || from > info.State.LastSeq { return nil, nil } + // The batch asked is what the log holds from there, never more: a pull for 20 entries where fewer exist waits its whole + // MaxWait (2 s) for the rest, and the ageing reads three retentions at every step's end. + if from < info.State.FirstSeq { + from = info.State.FirstSeq + } + if held := info.State.LastSeq - from + 1; held < uint64(limit) { + limit = int(held) + } cons, err := st.OrderedConsumer(l.ctx, jetstream.OrderedConsumerConfig{DeliverPolicy: jetstream.DeliverByStartSequencePolicy, OptStartSeq: from}) if err != nil { return nil, err diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steplog_read_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steplog_read_integration_test.go new file mode 100644 index 00000000..f523cbd4 --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steplog_read_integration_test.go @@ -0,0 +1,48 @@ +//go:build integration + +package main + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/testbus" +) + +func TestStepLogReadDoesNotWaitForABatchItCannotFill(t *testing.T) { + nc, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + js, _ := jetstream.New(nc) + ctx := context.Background() + const module = "m" + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(module, seat.StateStepLog), Subjects: []string{buscompose.LogSubject(module, seat.StateStepLog) + ".>"}}); err != nil { + t.Fatal(err) + } + l := stepLog{ctx: ctx, js: js, module: module} + for i := 0; i < 2000; i++ { + if err := l.Append(seat.LogEntry{Pipeline: fmt.Sprintf("main.%d.x", i), Step: "lint", What: seat.LogEntryWhatEnded, At: time.Now()}); err != nil { + t.Fatal(err) + } + } + // A read of the entries the log holds from a sequence returns at once, whether it holds more than the batch asked, fewer or + // none: the ageing reads it three times at every step's end, and a pull for a batch the log cannot fill waited 2 s each. + for _, c := range []struct { + from uint64 + want int + }{{1, 20}, {1990, 11}, {2001, 0}, {5000, 0}} { + t0 := time.Now() + got, err := l.Read(c.from, 20) + if err != nil || len(got) != c.want || time.Since(t0) > time.Second { + t.Errorf("Read(%d, 20) = %d entries (%v) in %v: want %d entries within a second", c.from, len(got), err, time.Since(t0), c.want) + } + } +} -- 2.54.0