diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 52bc05df1..6544c905f 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -376,7 +376,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err := hearRecovery(heard, kvRecoveries{ctx, recovered}, logger.Printf); err != nil { return err } - if err := hearTakenIn(heard, starter, rv.Catalog, logger.Printf); err != nil { + // take-in.taken is acted on only on the runner's own takeins durable, so no step traffic holds a take-in; the main + // durable acknowledges it at once (design 47 H2 H-A-500 (b) and its addenda). + if err := dropTakenIn(heard); err != nil { + return err + } + takeIns := &oneHandler{} + takeInsName := cred.EventConsumerName() + "_" + takeInsSuffix + takeInsHeard := agedHub{hub: takeIns, ledger: process.NewConsumerLedger(process.EventsStream, takeInsName, consumersSink(consumers), nil, logger.Printf)} + if err := hearTakenIn(takeInsHeard, starter, rv.Catalog, logger.Printf); err != nil { return err } if err := hearReleased(heard, hearer, starter.Registration, logger.Printf); err != nil { @@ -393,6 +401,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // their callers against (A-186): registered before the hub starts, read again once its events are live. seats := seatlist.NewPart("runner") seats.Follow(seatlist.ProcessBus(calls.Call, hub), nil, logger.Printf) + bindTakeIns(ctx, js, cred, takeIns.handle, waitToBind, logger.Printf) if err := hub.StartUntilBound(ctx, js, cred, nil, seats.EventsLive); err != nil { logger.Printf("the consumer %s is not bound yet (%v): bound once the bus makes it", cred.EventConsumerName(), err) } diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go index ff99ba24c..fd539a64b 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go @@ -172,6 +172,7 @@ func testTheQueueRunsPipelinesInTurn(t *testing.T) { return "" }, logs) }() + takeInsHeard(t, ctx, logs) return func() { stop(); <-done } } stopRunner := start() diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go index 3e051bedf..6b6c99278 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go @@ -52,6 +52,7 @@ func (s *syncBuffer) String() string { s.mu.Lock(); defer s.mu.Unlock(); return // bus proved; the pipeline then ends passed. func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *testing.T) { t.Run("the queue runs pipelines in turn", testTheQueueRunsPipelinesInTurn) + t.Run("a take-in is heard while late events queue on the main durable", testATakeInIsHeardWhileLateEventsQueue) t.Run("step log read does not wait for a batch it cannot fill", testStepLogReadDoesNotWaitForABatchItCannotFill) ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() @@ -384,6 +385,7 @@ func startWhole(t *testing.T, ctx context.Context, regs []catalog.Registration, }, logs) }() t.Cleanup(func() { stop(); <-done }) + takeInsHeard(t, ctx, logs) pipelines, err := js.KeyValue(ctx, module+"_pipelines") if err != nil { t.Fatal(err) @@ -500,6 +502,13 @@ func findingsStream(t *testing.T, ctx context.Context, js jetstream.JetStream, m } } +// takeInsHeard waits until the runner's takeins durable is bound, which the runner makes at its first start from the +// moment it is made (H-A-500 (b)): a take-in said before is heard by no one, as before the runner's first start. +func takeInsHeard(t *testing.T, ctx context.Context, logs *syncBuffer) { + t.Helper() + waitFor(t, ctx, logs, func() bool { return strings.Contains(logs.String(), "take-ins: heard on the consumer") }) +} + func waitFor(t *testing.T, ctx context.Context, logs *syncBuffer, cond func() bool) { t.Helper() for ctx.Err() == nil { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go index 1f9365cea..6e6caa4da 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go @@ -3,12 +3,17 @@ package main import ( "context" "encoding/json" + "errors" + "fmt" + "time" "github.com/nats-io/nats.go/jetstream" builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/process" + "mesh-delivery-pipeline/internal/pipeline" ) @@ -57,6 +62,98 @@ func revisedHear(revised func(ev catalog.CatalogRevisionEvent) (int, error), log } } +// takeInsSuffix names the runner's own second durable on the shared events stream, `__takeins`, on which alone +// take-in.taken is acted on, so a pipeline starts within seconds of its take-in whatever the step traffic on the runner's +// main durable (design 47 H2 H-A-500 (b) and its addenda). +const takeInsSuffix = "takeins" + +// The ack wait and the deliveries of the takeins durable, as the grants part gives a module's consumers (H-A-500 (b)). +const ( + takeInsAckWait = 30 * time.Second + takeInsMaxDeliver = 5 +) + +// takeInsConsumerConfig is the takeins durable as the runner makes it: take-in.taken alone, acknowledged one by one, from +// the moment it is made (H-A-500 (b)). +func takeInsConsumerConfig(cred process.Account) jetstream.ConsumerConfig { + cfg := cred.SeatEventConsumerConfig([]string{pipeline.TakenInSubject}, false) + cfg.Durable = cred.EventConsumerName() + "_" + takeInsSuffix + cfg.AckWait, cfg.MaxDeliver = takeInsAckWait, takeInsMaxDeliver + cfg.DeliverPolicy = jetstream.DeliverNewPolicy + return cfg +} + +// oneHandler is the handlers of one durable that hears one pattern: the takeins durable's (H-A-500 (b)). +type oneHandler struct { + pattern string + handle func(jetstream.Msg) +} + +func (o *oneHandler) Handle(pattern string, handle func(jetstream.Msg)) error { + if o.handle != nil { + return fmt.Errorf("the durable hears %q already, so %q has no taker", o.pattern, pattern) + } + o.pattern, o.handle = pattern, handle + return nil +} + +// dropTakenIn has the main durable acknowledge each take-in.taken at once with no work, the takeins durable alone acting on +// it, so the finding event-unhandled stays clear (H-A-500 third addendum). +func dropTakenIn(hub handlers) error { + return hub.Handle(pipeline.TakenInSubject, func(m jetstream.Msg) { _ = m.Ack() }) +} + +// bindTakeIns binds the runner's takeins durable and hears it with handle, the durable made when the bus does not hold it +// (H-A-500 (b)): bound at once when it can be, so no take-in said after the runner's start is missed; a bind that fails is +// said and tried again after each wait until it binds or ctx ends, as the hub binds the main durable. +func bindTakeIns(ctx context.Context, js jetstream.JetStream, cred process.Account, handle func(jetstream.Msg), wait func(context.Context) bool, + logf func(string, ...any)) { + bind := func() error { + cons, err := cred.NamedEventConsumer(ctx, js, process.EventsStream, takeInsSuffix) + if errors.Is(err, jetstream.ErrConsumerNotFound) { + cfg := takeInsConsumerConfig(cred) + if cons, err = js.CreateConsumer(ctx, process.EventsStream, cfg); err != nil { + return fmt.Errorf("the consumer %s on %s is not there and is not made: %w", cfg.Durable, process.EventsStream, err) + } + logf("take-ins: the consumer %s on %s was not there: the runner made it (H-A-500 (b))", cfg.Durable, process.EventsStream) + } + if err != nil { + return err + } + _, err = process.HearEvents(cons, handle, logf) + return err + } + try := func() bool { + err := bind() + if err == nil { + logf("take-ins: heard on the consumer %s_%s (H-A-500 (b))", cred.EventConsumerName(), takeInsSuffix) + return true + } + logf("take-ins: the consumer %s_%s is not bound, so no take-in starts a pipeline yet; tried again: %v", cred.EventConsumerName(), takeInsSuffix, err) + return false + } + if try() { + return + } + go func() { + for wait(ctx) && !try() { + } + }() +} + +// waitToBind waits five seconds or for ctx before the takeins durable is bound again: the bus makes the grant when the +// runner's account is composed, which nothing the runner hears says first (as the event hub's StartUntilBound waits). +func waitToBind(ctx context.Context) bool { + t := time.NewTimer(5 * time.Second) + defer t.Stop() + select { + case <-ctx.Done(): + return false + case <-t.C: + return true + } +} + // hearTakenIn starts the pipeline of each take-in.taken heard (design 47 H ยง746): one per take-in, the message acknowledged // once it is written, and heard again when a write failed, so the key already written starts nothing twice. func hearTakenIn(hub handlers, starter *pipeline.Starter, v *pipeline.Views, logf func(string, ...any)) error { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/takeins_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/takeins_integration_test.go new file mode 100644 index 000000000..44a9f877e --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/takeins_integration_test.go @@ -0,0 +1,88 @@ +//go:build integration + +package main + +import ( + "context" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/testbus" + + "mesh-delivery-pipeline/internal/pipeline" +) + +// Design 47 H2 H-A-500 (b) and its addenda: a take-in.taken said while 500 late step-progress events wait on the runner's +// main durable, its handler held, is heard at once on the runner's own takeins durable, which the runner makes itself; the +// main durable acknowledges the take-in with no work. testATakeInIsHeardWhileLateEventsQueue is a row of +// TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview (the module's tests are folded under its budget). +func testATakeInIsHeardWhileLateEventsQueue(t *testing.T) { + url := testbus.URL(t) + nc, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + cred := process.Account{URL: url, Node: "n1", Module: "mesh-delivery-pipeline"} + events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}}) + if err != nil { + t.Fatal(err) + } + if _, err := events.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{Durable: cred.EventConsumerName(), AckPolicy: jetstream.AckExplicitPolicy}); err != nil { + t.Fatal(err) + } + const progress = "mesh.seat.node-build-go.event.step-progress.n1" + for i := 0; i < 500; i++ { + if _, err := js.Publish(ctx, progress, []byte(`{"ask":"late","phase":"run"}`)); err != nil { + t.Fatal(err) + } + } + // The main durable's step-progress handler is held until the test ends: every late event waits behind the first. + held := make(chan struct{}) + defer close(held) + hub := process.NewEventHub(t.Logf) + if err := hub.Handle(progress, func(m jetstream.Msg) { <-held; _ = m.Ack() }); err != nil { + t.Fatal(err) + } + if err := dropTakenIn(hub); err != nil { + t.Fatal(err) + } + if err := hub.Start(ctx, js, cred); err != nil { + t.Fatal(err) + } + defer hub.Stop() + heard := make(chan string, 1) + takeIns := &oneHandler{} + if err := takeIns.Handle(pipeline.TakenInSubject, func(m jetstream.Msg) { heard <- m.Subject(); _ = m.Ack() }); err != nil { + t.Fatal(err) + } + bindTakeIns(ctx, js, cred, takeIns.handle, func(context.Context) bool { return false }, t.Logf) + if _, err := js.Publish(ctx, "mesh.mod.mesh-delivery-indexer.event.take-in.taken", []byte(`{}`)); err != nil { + t.Fatal(err) + } + select { + case s := <-heard: + if s != "mesh.mod.mesh-delivery-indexer.event.take-in.taken" { + t.Errorf("the takeins durable heard %s", s) + } + case <-ctx.Done(): + t.Fatal("the take-in is not heard while late events wait on the main durable") + } + info, err := events.Consumer(ctx, cred.EventConsumerName()+"_"+takeInsSuffix) + if err != nil { + t.Fatalf("the takeins durable is not made: %v", err) + } + if f := info.CachedInfo().Config.FilterSubjects; len(f) != 1 || f[0] != pipeline.TakenInSubject { + t.Errorf("the takeins durable filters %v, want take-in.taken alone", f) + } +} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go index 69b901247..463f0cb1a 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go @@ -75,7 +75,10 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(module, seat.StateStepLog), Subjects: []string{buscompose.LogSubject(module, seat.StateStepLog) + ".>"}}); err != nil { t.Fatal(err) } - events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>", "mesh.stream.*.event.>"}}) + // The runner's findings ride its own stream, as its manifest declares, never EVENTS: on EVENTS each one would reach its + // main durable unhandled and write event-unhandled again, a flood a take-in on its own durable overtakes (H-A-500 (b)). + findingsStream(t, ctx, js, module) + events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}}) if err != nil { t.Fatal(err) } @@ -224,6 +227,7 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) go func() { done <- run(runCtx, func(k string) string { return env[k] }, logs) }() + takeInsHeard(t, ctx, logs) return func() { stop(); <-done } } stop := start()