Runner: take-in.taken is heard on a durable of its own
The runner heard every event on one serial durable, so a take-in waited behind hundreds of late step events and its pipeline started ten minutes late. The runner now makes its own durable <node>_<module>_takeins on EVENTS, filtered to take-in.taken, and acts on a take-in there alone; the main durable acknowledges a take-in at once with no work. The test-runs integration test keeps the runner's findings on their own stream, as the manifest declares: on EVENTS each reached the main durable unhandled and wrote event-unhandled again, a flood the take-in now overtakes. Builds: design 47 H2 H-A-500 (b) and its addenda (2), third addendum (Q2)
This commit is contained in:
@@ -376,7 +376,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err := hearRecovery(heard, kvRecoveries{ctx, recovered}, logger.Printf); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := hearTakenIn(heard, starter, rv.Catalog, logger.Printf); err != nil {
|
||||
// take-in.taken is acted on only on the runner's own takeins durable, so no step traffic holds a take-in; the main
|
||||
// durable acknowledges it at once (design 47 H2 H-A-500 (b) and its addenda).
|
||||
if err := dropTakenIn(heard); err != nil {
|
||||
return err
|
||||
}
|
||||
takeIns := &oneHandler{}
|
||||
takeInsName := cred.EventConsumerName() + "_" + takeInsSuffix
|
||||
takeInsHeard := agedHub{hub: takeIns, ledger: process.NewConsumerLedger(process.EventsStream, takeInsName, consumersSink(consumers), nil, logger.Printf)}
|
||||
if err := hearTakenIn(takeInsHeard, starter, rv.Catalog, logger.Printf); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := hearReleased(heard, hearer, starter.Registration, logger.Printf); err != nil {
|
||||
@@ -393,6 +401,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// their callers against (A-186): registered before the hub starts, read again once its events are live.
|
||||
seats := seatlist.NewPart("runner")
|
||||
seats.Follow(seatlist.ProcessBus(calls.Call, hub), nil, logger.Printf)
|
||||
bindTakeIns(ctx, js, cred, takeIns.handle, waitToBind, logger.Printf)
|
||||
if err := hub.StartUntilBound(ctx, js, cred, nil, seats.EventsLive); err != nil {
|
||||
logger.Printf("the consumer %s is not bound yet (%v): bound once the bus makes it", cred.EventConsumerName(), err)
|
||||
}
|
||||
|
||||
@@ -172,6 +172,7 @@ func testTheQueueRunsPipelinesInTurn(t *testing.T) {
|
||||
return ""
|
||||
}, logs)
|
||||
}()
|
||||
takeInsHeard(t, ctx, logs)
|
||||
return func() { stop(); <-done }
|
||||
}
|
||||
stopRunner := start()
|
||||
|
||||
@@ -52,6 +52,7 @@ func (s *syncBuffer) String() string { s.mu.Lock(); defer s.mu.Unlock(); return
|
||||
// bus proved; the pipeline then ends passed.
|
||||
func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *testing.T) {
|
||||
t.Run("the queue runs pipelines in turn", testTheQueueRunsPipelinesInTurn)
|
||||
t.Run("a take-in is heard while late events queue on the main durable", testATakeInIsHeardWhileLateEventsQueue)
|
||||
t.Run("step log read does not wait for a batch it cannot fill", testStepLogReadDoesNotWaitForABatchItCannotFill)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
@@ -384,6 +385,7 @@ func startWhole(t *testing.T, ctx context.Context, regs []catalog.Registration,
|
||||
}, logs)
|
||||
}()
|
||||
t.Cleanup(func() { stop(); <-done })
|
||||
takeInsHeard(t, ctx, logs)
|
||||
pipelines, err := js.KeyValue(ctx, module+"_pipelines")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -500,6 +502,13 @@ func findingsStream(t *testing.T, ctx context.Context, js jetstream.JetStream, m
|
||||
}
|
||||
}
|
||||
|
||||
// takeInsHeard waits until the runner's takeins durable is bound, which the runner makes at its first start from the
|
||||
// moment it is made (H-A-500 (b)): a take-in said before is heard by no one, as before the runner's first start.
|
||||
func takeInsHeard(t *testing.T, ctx context.Context, logs *syncBuffer) {
|
||||
t.Helper()
|
||||
waitFor(t, ctx, logs, func() bool { return strings.Contains(logs.String(), "take-ins: heard on the consumer") })
|
||||
}
|
||||
|
||||
func waitFor(t *testing.T, ctx context.Context, logs *syncBuffer, cond func() bool) {
|
||||
t.Helper()
|
||||
for ctx.Err() == nil {
|
||||
|
||||
@@ -3,12 +3,17 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-pipeline/internal/pipeline"
|
||||
)
|
||||
|
||||
@@ -57,6 +62,98 @@ func revisedHear(revised func(ev catalog.CatalogRevisionEvent) (int, error), log
|
||||
}
|
||||
}
|
||||
|
||||
// takeInsSuffix names the runner's own second durable on the shared events stream, `<node>_<module>_takeins`, on which alone
|
||||
// take-in.taken is acted on, so a pipeline starts within seconds of its take-in whatever the step traffic on the runner's
|
||||
// main durable (design 47 H2 H-A-500 (b) and its addenda).
|
||||
const takeInsSuffix = "takeins"
|
||||
|
||||
// The ack wait and the deliveries of the takeins durable, as the grants part gives a module's consumers (H-A-500 (b)).
|
||||
const (
|
||||
takeInsAckWait = 30 * time.Second
|
||||
takeInsMaxDeliver = 5
|
||||
)
|
||||
|
||||
// takeInsConsumerConfig is the takeins durable as the runner makes it: take-in.taken alone, acknowledged one by one, from
|
||||
// the moment it is made (H-A-500 (b)).
|
||||
func takeInsConsumerConfig(cred process.Account) jetstream.ConsumerConfig {
|
||||
cfg := cred.SeatEventConsumerConfig([]string{pipeline.TakenInSubject}, false)
|
||||
cfg.Durable = cred.EventConsumerName() + "_" + takeInsSuffix
|
||||
cfg.AckWait, cfg.MaxDeliver = takeInsAckWait, takeInsMaxDeliver
|
||||
cfg.DeliverPolicy = jetstream.DeliverNewPolicy
|
||||
return cfg
|
||||
}
|
||||
|
||||
// oneHandler is the handlers of one durable that hears one pattern: the takeins durable's (H-A-500 (b)).
|
||||
type oneHandler struct {
|
||||
pattern string
|
||||
handle func(jetstream.Msg)
|
||||
}
|
||||
|
||||
func (o *oneHandler) Handle(pattern string, handle func(jetstream.Msg)) error {
|
||||
if o.handle != nil {
|
||||
return fmt.Errorf("the durable hears %q already, so %q has no taker", o.pattern, pattern)
|
||||
}
|
||||
o.pattern, o.handle = pattern, handle
|
||||
return nil
|
||||
}
|
||||
|
||||
// dropTakenIn has the main durable acknowledge each take-in.taken at once with no work, the takeins durable alone acting on
|
||||
// it, so the finding event-unhandled stays clear (H-A-500 third addendum).
|
||||
func dropTakenIn(hub handlers) error {
|
||||
return hub.Handle(pipeline.TakenInSubject, func(m jetstream.Msg) { _ = m.Ack() })
|
||||
}
|
||||
|
||||
// bindTakeIns binds the runner's takeins durable and hears it with handle, the durable made when the bus does not hold it
|
||||
// (H-A-500 (b)): bound at once when it can be, so no take-in said after the runner's start is missed; a bind that fails is
|
||||
// said and tried again after each wait until it binds or ctx ends, as the hub binds the main durable.
|
||||
func bindTakeIns(ctx context.Context, js jetstream.JetStream, cred process.Account, handle func(jetstream.Msg), wait func(context.Context) bool,
|
||||
logf func(string, ...any)) {
|
||||
bind := func() error {
|
||||
cons, err := cred.NamedEventConsumer(ctx, js, process.EventsStream, takeInsSuffix)
|
||||
if errors.Is(err, jetstream.ErrConsumerNotFound) {
|
||||
cfg := takeInsConsumerConfig(cred)
|
||||
if cons, err = js.CreateConsumer(ctx, process.EventsStream, cfg); err != nil {
|
||||
return fmt.Errorf("the consumer %s on %s is not there and is not made: %w", cfg.Durable, process.EventsStream, err)
|
||||
}
|
||||
logf("take-ins: the consumer %s on %s was not there: the runner made it (H-A-500 (b))", cfg.Durable, process.EventsStream)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = process.HearEvents(cons, handle, logf)
|
||||
return err
|
||||
}
|
||||
try := func() bool {
|
||||
err := bind()
|
||||
if err == nil {
|
||||
logf("take-ins: heard on the consumer %s_%s (H-A-500 (b))", cred.EventConsumerName(), takeInsSuffix)
|
||||
return true
|
||||
}
|
||||
logf("take-ins: the consumer %s_%s is not bound, so no take-in starts a pipeline yet; tried again: %v", cred.EventConsumerName(), takeInsSuffix, err)
|
||||
return false
|
||||
}
|
||||
if try() {
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
for wait(ctx) && !try() {
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// waitToBind waits five seconds or for ctx before the takeins durable is bound again: the bus makes the grant when the
|
||||
// runner's account is composed, which nothing the runner hears says first (as the event hub's StartUntilBound waits).
|
||||
func waitToBind(ctx context.Context) bool {
|
||||
t := time.NewTimer(5 * time.Second)
|
||||
defer t.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-t.C:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// hearTakenIn starts the pipeline of each take-in.taken heard (design 47 H §746): one per take-in, the message acknowledged
|
||||
// once it is written, and heard again when a write failed, so the key already written starts nothing twice.
|
||||
func hearTakenIn(hub handlers, starter *pipeline.Starter, v *pipeline.Views, logf func(string, ...any)) error {
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/testbus"
|
||||
|
||||
"mesh-delivery-pipeline/internal/pipeline"
|
||||
)
|
||||
|
||||
// Design 47 H2 H-A-500 (b) and its addenda: a take-in.taken said while 500 late step-progress events wait on the runner's
|
||||
// main durable, its handler held, is heard at once on the runner's own takeins durable, which the runner makes itself; the
|
||||
// main durable acknowledges the take-in with no work. testATakeInIsHeardWhileLateEventsQueue is a row of
|
||||
// TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview (the module's tests are folded under its budget).
|
||||
func testATakeInIsHeardWhileLateEventsQueue(t *testing.T) {
|
||||
url := testbus.URL(t)
|
||||
nc, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer nc.Close()
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
cred := process.Account{URL: url, Node: "n1", Module: "mesh-delivery-pipeline"}
|
||||
events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := events.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{Durable: cred.EventConsumerName(), AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const progress = "mesh.seat.node-build-go.event.step-progress.n1"
|
||||
for i := 0; i < 500; i++ {
|
||||
if _, err := js.Publish(ctx, progress, []byte(`{"ask":"late","phase":"run"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// The main durable's step-progress handler is held until the test ends: every late event waits behind the first.
|
||||
held := make(chan struct{})
|
||||
defer close(held)
|
||||
hub := process.NewEventHub(t.Logf)
|
||||
if err := hub.Handle(progress, func(m jetstream.Msg) { <-held; _ = m.Ack() }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := dropTakenIn(hub); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := hub.Start(ctx, js, cred); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer hub.Stop()
|
||||
heard := make(chan string, 1)
|
||||
takeIns := &oneHandler{}
|
||||
if err := takeIns.Handle(pipeline.TakenInSubject, func(m jetstream.Msg) { heard <- m.Subject(); _ = m.Ack() }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bindTakeIns(ctx, js, cred, takeIns.handle, func(context.Context) bool { return false }, t.Logf)
|
||||
if _, err := js.Publish(ctx, "mesh.mod.mesh-delivery-indexer.event.take-in.taken", []byte(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case s := <-heard:
|
||||
if s != "mesh.mod.mesh-delivery-indexer.event.take-in.taken" {
|
||||
t.Errorf("the takeins durable heard %s", s)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Fatal("the take-in is not heard while late events wait on the main durable")
|
||||
}
|
||||
info, err := events.Consumer(ctx, cred.EventConsumerName()+"_"+takeInsSuffix)
|
||||
if err != nil {
|
||||
t.Fatalf("the takeins durable is not made: %v", err)
|
||||
}
|
||||
if f := info.CachedInfo().Config.FilterSubjects; len(f) != 1 || f[0] != pipeline.TakenInSubject {
|
||||
t.Errorf("the takeins durable filters %v, want take-in.taken alone", f)
|
||||
}
|
||||
}
|
||||
+5
-1
@@ -75,7 +75,10 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(module, seat.StateStepLog), Subjects: []string{buscompose.LogSubject(module, seat.StateStepLog) + ".>"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>", "mesh.stream.*.event.>"}})
|
||||
// The runner's findings ride its own stream, as its manifest declares, never EVENTS: on EVENTS each one would reach its
|
||||
// main durable unhandled and write event-unhandled again, a flood a take-in on its own durable overtakes (H-A-500 (b)).
|
||||
findingsStream(t, ctx, js, module)
|
||||
events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -224,6 +227,7 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
|
||||
runCtx, stop := context.WithCancel(ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- run(runCtx, func(k string) string { return env[k] }, logs) }()
|
||||
takeInsHeard(t, ctx, logs)
|
||||
return func() { stop(); <-done }
|
||||
}
|
||||
stop := start()
|
||||
|
||||
Reference in New Issue
Block a user