diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 184898496..bea4010f4 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -237,8 +237,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } } advancer.Store(&advance) - // A registration-changed places the steps of its module's pipelines started before it, from it (A-327). + // A registration-changed places the steps of its module's pipelines started before it, from it (A-327), and binds the + // takeins durable when its bind failed at the start (H-A-500 fifth addendum). + var bindTakeInsAgain atomic.Pointer[func()] register := func(reg catalog.Registration) { + if f := bindTakeInsAgain.Load(); f != nil { + (*f)() + } if n, err := starter.Registered(store, reg, rv.Catalog.View); err != nil || n > 0 { logger.Printf("the pipelines of %s started before its registration: %d placed (%v)", reg.Module, n, err) } @@ -403,7 +408,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // their callers against (A-186): registered before the hub starts, read again once its events are live. seats := seatlist.NewPart("runner") seats.Follow(seatlist.ProcessBus(calls.Call, hub), nil, logger.Printf) - bindTakeIns(ctx, js, cred, takeIns.handle, takeInsBound, waitToBind, logger.Printf) + again := bindTakeIns(ctx, js, cred, takeIns.handle, takeInsBound, logger.Printf) + bindTakeInsAgain.Store(&again) if err := hub.StartUntilBound(ctx, js, cred, nil, seats.EventsLive); err != nil { logger.Printf("the consumer %s is not bound yet (%v): bound once the bus makes it", cred.EventConsumerName(), err) } diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go index 00a6bd530..58272cbd4 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go @@ -54,6 +54,7 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes t.Run("the queue runs pipelines in turn", testTheQueueRunsPipelinesInTurn) t.Run("a take-in is heard while late events queue on the main durable", testATakeInIsHeardWhileLateEventsQueue) t.Run("a take-in said before the bind is still worked", testATakeInSaidBeforeTheBindIsStillWorked) + t.Run("a takeins bind that failed is bound again on registration-changed", testATakeInsBindThatFailedIsBoundAgainOnRegistrationChanged) t.Run("step log read does not wait for a batch it cannot fill", testStepLogReadDoesNotWaitForABatchItCannotFill) ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go index b65dd4c38..59fd514ea 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "sync" "sync/atomic" "time" @@ -138,12 +139,13 @@ func dropTakenIn(hub handlers, sw *takeInsSwitch, work func(jetstream.Msg)) erro } // bindTakeIns binds the runner's takeins durable and hears it with handle, the durable made when the bus does not hold it -// (H-A-500 (b)): bound at once when it can be, so no take-in said after the runner's start is missed; a bind that fails is -// said and tried again after each wait until it binds or ctx ends, as the hub binds the main durable. Once bound, sw says -// so with the stream sequence the durable delivers after, and the main durable stops working the take-ins it delivers -// (H-A-500 fourth addendum). +// (H-A-500 (b)): bound at once when it can be, so no take-in said after the runner's start is missed. A bind that fails +// (the bus has not made the runner's grant yet) is said, and the answered func binds again, called on each +// registration-changed as the indexer binds a branch's consumer on each (H-A-500 fifth addendum: nothing waits on a timer +// for another part); once bound it does nothing. Once bound, sw says so with the stream sequence the durable delivers +// after, and the main durable stops working the take-ins it delivers (the fourth addendum). func bindTakeIns(ctx context.Context, js jetstream.JetStream, cred process.Account, handle func(jetstream.Msg), sw *takeInsSwitch, - wait func(context.Context) bool, logf func(string, ...any)) { + logf func(string, ...any)) (again func()) { bind := func() error { cons, err := cred.NamedEventConsumer(ctx, js, process.EventsStream, takeInsSuffix) if errors.Is(err, jetstream.ErrConsumerNotFound) { @@ -167,35 +169,24 @@ func bindTakeIns(ctx context.Context, js jetstream.JetStream, cred process.Accou sw.boundFrom(info.Delivered.Stream) return nil } - try := func() bool { - err := bind() - if err == nil { - logf("take-ins: heard on the consumer %s_%s (H-A-500 (b))", cred.EventConsumerName(), takeInsSuffix) - return true + var mu sync.Mutex + bound := false + try := func() { + mu.Lock() + defer mu.Unlock() + if bound || ctx.Err() != nil { + return } - logf("take-ins: the consumer %s_%s is not bound, so no take-in starts a pipeline yet; tried again: %v", cred.EventConsumerName(), takeInsSuffix, err) - return false - } - if try() { - return - } - go func() { - for wait(ctx) && !try() { + if err := bind(); err != nil { + logf("take-ins: the consumer %s_%s is not bound, so the main durable works each take-in; bound again at the next registration-changed: %v", + cred.EventConsumerName(), takeInsSuffix, err) + return } - }() -} - -// waitToBind waits five seconds or for ctx before the takeins durable is bound again: the bus makes the grant when the -// runner's account is composed, which nothing the runner hears says first (as the event hub's StartUntilBound waits). -func waitToBind(ctx context.Context) bool { - t := time.NewTimer(5 * time.Second) - defer t.Stop() - select { - case <-ctx.Done(): - return false - case <-t.C: - return true + bound = true + logf("take-ins: heard on the consumer %s_%s (H-A-500 (b))", cred.EventConsumerName(), takeInsSuffix) } + try() + return try } // hearTakenIn starts the pipeline of each take-in.taken heard (design 47 H ยง746): one per take-in, the message acknowledged diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/takeins_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/takeins_integration_test.go index ee01ce0eb..a96c4f9e2 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/takeins_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/takeins_integration_test.go @@ -67,7 +67,7 @@ func testATakeInIsHeardWhileLateEventsQueue(t *testing.T) { if err := takeIns.Handle(pipeline.TakenInSubject, func(m jetstream.Msg) { heard <- m.Subject(); _ = m.Ack() }); err != nil { t.Fatal(err) } - bindTakeIns(ctx, js, cred, takeIns.handle, sw, func(context.Context) bool { return false }, t.Logf) + bindTakeIns(ctx, js, cred, takeIns.handle, sw, t.Logf) if _, err := js.Publish(ctx, "mesh.mod.mesh-delivery-indexer.event.take-in.taken", []byte(`{}`)); err != nil { t.Fatal(err) } @@ -135,7 +135,7 @@ func testATakeInSaidBeforeTheBindIsStillWorked(t *testing.T) { if err := takeIns.Handle(pipeline.TakenInSubject, func(m jetstream.Msg) { heard <- string(m.Data()); _ = m.Ack() }); err != nil { t.Fatal(err) } - bindTakeIns(ctx, js, cred, takeIns.handle, sw, func(context.Context) bool { return false }, t.Logf) + bindTakeIns(ctx, js, cred, takeIns.handle, sw, t.Logf) if sw.from.Load() == 0 { t.Fatal("the takeins durable is not bound") } @@ -178,3 +178,44 @@ func testATakeInSaidBeforeTheBindIsStillWorked(t *testing.T) { t.Errorf("the main durable holds %d pending and %d unacknowledged, want none", ci.NumPending, ci.NumAckPending) } } + +// Design 47 H2 H-A-500 fifth addendum: a takeins bind that failed at the start (here the events stream is not there yet, as +// the runner's grant is not there before the bus composes its account) is said and bound again on the next +// registration-changed, never on a timer. A row of TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview. +func testATakeInsBindThatFailedIsBoundAgainOnRegistrationChanged(t *testing.T) { + url := testbus.URL(t) + nc, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + cred := process.Account{URL: url, Node: "n1", Module: "mesh-delivery-pipeline"} + sw := &takeInsSwitch{} + heard := make(chan string, 1) + again := bindTakeIns(ctx, js, cred, func(m jetstream.Msg) { heard <- m.Subject(); _ = m.Ack() }, sw, t.Logf) + if sw.from.Load() != 0 { + t.Fatal("the takeins durable is bound with no events stream") + } + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}}); err != nil { + t.Fatal(err) + } + again() // the registration-changed heard once the bus holds the stream and the grant + if sw.from.Load() == 0 { + t.Fatal("the takeins durable is not bound again on registration-changed") + } + if _, err := js.Publish(ctx, "mesh.mod.mesh-delivery-indexer.event.take-in.taken", []byte(`{}`)); err != nil { + t.Fatal(err) + } + select { + case <-heard: + case <-ctx.Done(): + t.Fatal("the take-in is not heard on the durable bound again") + } + again() // bound already: nothing is bound twice +}