diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/consumers_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/consumers_test.go index 91ddcd8ca..6b3a55a29 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/consumers_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/consumers_test.go @@ -60,3 +60,37 @@ func TestAgedHub(t *testing.T) { t.Errorf("a message asked again is noted applied: %+v", last) } } + +// TestTheTakeInsSwitch: until the takeins durable is bound, the main durable works every take-in; from the bind, it works +// only those said at or before the sequence the durable delivers after, and acknowledges the rest with no work +// (design 47 H2 H-A-500 fourth addendum). +func TestTheTakeInsSwitch(t *testing.T) { + for _, row := range []struct { + name string + boundFrom uint64 + bound bool + seq uint64 + delivers bool + }{ + {"not bound, any sequence is worked", 0, false, 7, false}, + {"bound after 5, the take-in at 5 is worked", 5, true, 5, false}, + {"bound after 5, the take-in at 6 is the durable's", 5, true, 6, true}, + {"bound on an empty stream, the first take-in is the durable's", 0, true, 1, true}, + } { + t.Run(row.name, func(t *testing.T) { + sw := &takeInsSwitch{} + if row.bound { + sw.boundFrom(row.boundFrom) + } + hub := &oneHub{} + worked := false + if err := dropTakenIn(hub, sw, func(jetstream.Msg) { worked = true }); err != nil { + t.Fatal(err) + } + hub.handle(ackedMsg{seq: row.seq}) + if worked == row.delivers { + t.Errorf("worked by the main durable: %v, want %v", worked, !row.delivers) + } + }) + } +} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 6544c905f..184898496 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -376,9 +376,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err := hearRecovery(heard, kvRecoveries{ctx, recovered}, logger.Printf); err != nil { return err } - // take-in.taken is acted on only on the runner's own takeins durable, so no step traffic holds a take-in; the main - // durable acknowledges it at once (design 47 H2 H-A-500 (b) and its addenda). - if err := dropTakenIn(heard); err != nil { + // take-in.taken is acted on on the runner's own takeins durable once that is bound, so no step traffic holds a take-in; + // until then the main durable works it, and from the bind acknowledges it at once (design 47 H2 H-A-500 (b) and its + // addenda, the fourth: no take-in said while the runner was down is lost). + takeInsBound := &takeInsSwitch{} + if err := dropTakenIn(heard, takeInsBound, takenIn(starter, rv.Catalog, logger.Printf)); err != nil { return err } takeIns := &oneHandler{} @@ -401,7 +403,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // their callers against (A-186): registered before the hub starts, read again once its events are live. seats := seatlist.NewPart("runner") seats.Follow(seatlist.ProcessBus(calls.Call, hub), nil, logger.Printf) - bindTakeIns(ctx, js, cred, takeIns.handle, waitToBind, logger.Printf) + bindTakeIns(ctx, js, cred, takeIns.handle, takeInsBound, waitToBind, logger.Printf) if err := hub.StartUntilBound(ctx, js, cred, nil, seats.EventsLive); err != nil { logger.Printf("the consumer %s is not bound yet (%v): bound once the bus makes it", cred.EventConsumerName(), err) } diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go index 6b6c99278..00a6bd530 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go @@ -53,6 +53,7 @@ func (s *syncBuffer) String() string { s.mu.Lock(); defer s.mu.Unlock(); return func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *testing.T) { t.Run("the queue runs pipelines in turn", testTheQueueRunsPipelinesInTurn) t.Run("a take-in is heard while late events queue on the main durable", testATakeInIsHeardWhileLateEventsQueue) + t.Run("a take-in said before the bind is still worked", testATakeInSaidBeforeTheBindIsStillWorked) t.Run("step log read does not wait for a batch it cannot fill", testStepLogReadDoesNotWaitForABatchItCannotFill) ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() @@ -503,7 +504,8 @@ func findingsStream(t *testing.T, ctx context.Context, js jetstream.JetStream, m } // takeInsHeard waits until the runner's takeins durable is bound, which the runner makes at its first start from the -// moment it is made (H-A-500 (b)): a take-in said before is heard by no one, as before the runner's first start. +// moment it is made (H-A-500 (b)); a take-in said before is worked by the main durable (the fourth addendum), but the tests +// publish after the bind so that each take-in is heard where the running runner hears it. func takeInsHeard(t *testing.T, ctx context.Context, logs *syncBuffer) { t.Helper() waitFor(t, ctx, logs, func() bool { return strings.Contains(logs.String(), "take-ins: heard on the consumer") }) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go index 6e6caa4da..b65dd4c38 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "sync/atomic" "time" "github.com/nats-io/nats.go/jetstream" @@ -97,17 +98,52 @@ func (o *oneHandler) Handle(pattern string, handle func(jetstream.Msg)) error { return nil } -// dropTakenIn has the main durable acknowledge each take-in.taken at once with no work, the takeins durable alone acting on -// it, so the finding event-unhandled stays clear (H-A-500 third addendum). -func dropTakenIn(hub handlers) error { - return hub.Handle(pipeline.TakenInSubject, func(m jetstream.Msg) { _ = m.Ack() }) +// takeInsSwitch says which durable a take-in.taken is worked on (H-A-500 fourth addendum): until the runner's takeins +// durable is bound, the main durable does the work, so a take-in said while the runner was down is lost to no one; from +// the bind, the takeins durable alone works the take-ins said after it was made, and the main durable acknowledges those +// at once with no work. The boundary is the stream sequence the takeins durable delivers from, so a take-in said before +// the durable was made and still waiting on the main durable is worked there whatever the moment it is heard. +type takeInsSwitch struct { + // from holds 1 + the stream sequence the takeins durable delivers after; 0 while it is not bound. + from atomic.Uint64 +} + +// boundFrom says the takeins durable is bound and delivers every take-in said after stream sequence seq. +func (s *takeInsSwitch) boundFrom(seq uint64) { s.from.Store(seq + 1) } + +// delivers says whether the takeins durable delivers m, so the main durable has no work on it. +func (s *takeInsSwitch) delivers(m jetstream.Msg) bool { + from := s.from.Load() + if from == 0 { + return false + } + meta, err := m.Metadata() + if err != nil { + // A message whose sequence is unknown is worked rather than dropped: the work is done once per key, never twice. + return false + } + return meta.Sequence.Stream >= from +} + +// dropTakenIn has the main durable work each take-in.taken the takeins durable does not deliver, and acknowledge the rest at +// once with no work, so the finding event-unhandled stays clear (H-A-500 third and fourth addenda). +func dropTakenIn(hub handlers, sw *takeInsSwitch, work func(jetstream.Msg)) error { + return hub.Handle(pipeline.TakenInSubject, func(m jetstream.Msg) { + if sw.delivers(m) { + _ = m.Ack() + return + } + work(m) + }) } // bindTakeIns binds the runner's takeins durable and hears it with handle, the durable made when the bus does not hold it // (H-A-500 (b)): bound at once when it can be, so no take-in said after the runner's start is missed; a bind that fails is -// said and tried again after each wait until it binds or ctx ends, as the hub binds the main durable. -func bindTakeIns(ctx context.Context, js jetstream.JetStream, cred process.Account, handle func(jetstream.Msg), wait func(context.Context) bool, - logf func(string, ...any)) { +// said and tried again after each wait until it binds or ctx ends, as the hub binds the main durable. Once bound, sw says +// so with the stream sequence the durable delivers after, and the main durable stops working the take-ins it delivers +// (H-A-500 fourth addendum). +func bindTakeIns(ctx context.Context, js jetstream.JetStream, cred process.Account, handle func(jetstream.Msg), sw *takeInsSwitch, + wait func(context.Context) bool, logf func(string, ...any)) { bind := func() error { cons, err := cred.NamedEventConsumer(ctx, js, process.EventsStream, takeInsSuffix) if errors.Is(err, jetstream.ErrConsumerNotFound) { @@ -120,8 +156,16 @@ func bindTakeIns(ctx context.Context, js jetstream.JetStream, cred process.Accou if err != nil { return err } - _, err = process.HearEvents(cons, handle, logf) - return err + // A durable just made delivers after the stream's last sequence; one found, after what it delivered already. + info, err := cons.Info(ctx) + if err != nil { + return fmt.Errorf("the consumer %s_%s is not read: %w", cred.EventConsumerName(), takeInsSuffix, err) + } + if _, err = process.HearEvents(cons, handle, logf); err != nil { + return err + } + sw.boundFrom(info.Delivered.Stream) + return nil } try := func() bool { err := bind() @@ -157,7 +201,12 @@ func waitToBind(ctx context.Context) bool { // hearTakenIn starts the pipeline of each take-in.taken heard (design 47 H §746): one per take-in, the message acknowledged // once it is written, and heard again when a write failed, so the key already written starts nothing twice. func hearTakenIn(hub handlers, starter *pipeline.Starter, v *pipeline.Views, logf func(string, ...any)) error { - return hub.Handle(pipeline.TakenInSubject, func(m jetstream.Msg) { + return hub.Handle(pipeline.TakenInSubject, takenIn(starter, v, logf)) +} + +// takenIn is the work of one take-in.taken, on whichever durable delivers it (H-A-500 fourth addendum). +func takenIn(starter *pipeline.Starter, v *pipeline.Views, logf func(string, ...any)) func(jetstream.Msg) { + return func(m jetstream.Msg) { // An event of the old form is refused by the contract's strict decode, said in one journal line and ended (§765, H-A-98). ev, err := pipeline.DecodeTakenIn(m.Data()) if err != nil { @@ -173,7 +222,7 @@ func hearTakenIn(hub handlers, starter *pipeline.Starter, v *pipeline.Views, log } logf("the take-in of %s source %d started %d pipelines", ev.Branch, ev.Source, n) _ = m.Ack() - }) + } } // hearReleased writes each version-released's version into its mover (design 47 H §746, H-A-52), so the pipeline's end names diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/takeins_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/takeins_integration_test.go index 44a9f877e..ee01ce0eb 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/takeins_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/takeins_integration_test.go @@ -54,7 +54,8 @@ func testATakeInIsHeardWhileLateEventsQueue(t *testing.T) { if err := hub.Handle(progress, func(m jetstream.Msg) { <-held; _ = m.Ack() }); err != nil { t.Fatal(err) } - if err := dropTakenIn(hub); err != nil { + sw := &takeInsSwitch{} + if err := dropTakenIn(hub, sw, func(m jetstream.Msg) { t.Errorf("the main durable worked %s after the bind", m.Subject()); _ = m.Ack() }); err != nil { t.Fatal(err) } if err := hub.Start(ctx, js, cred); err != nil { @@ -66,7 +67,7 @@ func testATakeInIsHeardWhileLateEventsQueue(t *testing.T) { if err := takeIns.Handle(pipeline.TakenInSubject, func(m jetstream.Msg) { heard <- m.Subject(); _ = m.Ack() }); err != nil { t.Fatal(err) } - bindTakeIns(ctx, js, cred, takeIns.handle, func(context.Context) bool { return false }, t.Logf) + bindTakeIns(ctx, js, cred, takeIns.handle, sw, func(context.Context) bool { return false }, t.Logf) if _, err := js.Publish(ctx, "mesh.mod.mesh-delivery-indexer.event.take-in.taken", []byte(`{}`)); err != nil { t.Fatal(err) } @@ -86,3 +87,94 @@ func testATakeInIsHeardWhileLateEventsQueue(t *testing.T) { t.Errorf("the takeins durable filters %v, want take-in.taken alone", f) } } + +// Design 47 H2 H-A-500 fourth addendum: a take-in.taken said before the runner's takeins durable is bound (the runner was +// down, or the bind failed) is worked by the main durable, so it starts its pipeline; one said after the bind is worked by +// the takeins durable alone and the main durable acknowledges it with no work. A row of +// TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview. +func testATakeInSaidBeforeTheBindIsStillWorked(t *testing.T) { + url := testbus.URL(t) + nc, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + cred := process.Account{URL: url, Node: "n1", Module: "mesh-delivery-pipeline"} + events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}}) + if err != nil { + t.Fatal(err) + } + if _, err := events.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{Durable: cred.EventConsumerName(), AckPolicy: jetstream.AckExplicitPolicy}); err != nil { + t.Fatal(err) + } + const subject = "mesh.mod.mesh-delivery-indexer.event.take-in.taken" + // Said while the runner was down: the main durable holds it, the takeins durable does not exist yet. + if _, err := js.Publish(ctx, subject, []byte(`{"before":true}`)); err != nil { + t.Fatal(err) + } + // The main durable's take-in handler is held until the takeins durable is bound, as a main durable busy with step + // events would be: the take-in said before the bind is still worked, by its sequence, not by the moment it is heard. + held := make(chan struct{}) + worked, heard := make(chan string, 2), make(chan string, 2) + hub := process.NewEventHub(t.Logf) + sw := &takeInsSwitch{} + if err := dropTakenIn(hub, sw, func(m jetstream.Msg) { <-held; worked <- string(m.Data()); _ = m.Ack() }); err != nil { + t.Fatal(err) + } + if err := hub.Start(ctx, js, cred); err != nil { + t.Fatal(err) + } + defer hub.Stop() + takeIns := &oneHandler{} + if err := takeIns.Handle(pipeline.TakenInSubject, func(m jetstream.Msg) { heard <- string(m.Data()); _ = m.Ack() }); err != nil { + t.Fatal(err) + } + bindTakeIns(ctx, js, cred, takeIns.handle, sw, func(context.Context) bool { return false }, t.Logf) + if sw.from.Load() == 0 { + t.Fatal("the takeins durable is not bound") + } + // Said after the bind: the takeins durable's alone. + if _, err := js.Publish(ctx, subject, []byte(`{"after":true}`)); err != nil { + t.Fatal(err) + } + close(held) + select { + case d := <-worked: + if d != `{"before":true}` { + t.Errorf("the main durable worked %s, want the take-in said before the bind", d) + } + case <-ctx.Done(): + t.Fatal("the take-in said before the bind is worked by no durable") + } + select { + case d := <-heard: + if d != `{"after":true}` { + t.Errorf("the takeins durable heard %s, want the take-in said after the bind", d) + } + case <-ctx.Done(): + t.Fatal("the take-in said after the bind is not heard on the takeins durable") + } + // The main durable acknowledged the second with no work: nothing more is worked once its queue is drained. + select { + case d := <-worked: + t.Errorf("the main durable worked %s after the bind", d) + case <-time.After(2 * time.Second): + } + info, err := events.Consumer(ctx, cred.EventConsumerName()) + if err != nil { + t.Fatal(err) + } + ci, err := info.Info(ctx) + if err != nil { + t.Fatal(err) + } + if ci.NumAckPending != 0 || ci.NumPending != 0 { + t.Errorf("the main durable holds %d pending and %d unacknowledged, want none", ci.NumPending, ci.NumAckPending) + } +}