Runner: take-in.taken is heard on a durable of its own

The runner heard every event on one serial durable, so a take-in waited behind
hundreds of late step events and its pipeline started ten minutes late. The
runner now makes its own durable <node>_<module>_takeins on EVENTS, filtered
to take-in.taken, and acts on a take-in there alone; the main durable
acknowledges a take-in at once with no work. The test-runs integration test
keeps the runner's findings on their own stream, as the manifest declares:
on EVENTS each reached the main durable unhandled and wrote event-unhandled
again, a flood the take-in now overtakes.

Builds: design 47 H2 H-A-500 (b) and its addenda (2), third addendum (Q2)
This commit is contained in:
2026-11-22 22:53:04 +01:00
parent 6ffddedfbf
commit b2d9d1ac90
6 changed files with 210 additions and 2 deletions
@@ -376,7 +376,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err := hearRecovery(heard, kvRecoveries{ctx, recovered}, logger.Printf); err != nil {
return err
}
if err := hearTakenIn(heard, starter, rv.Catalog, logger.Printf); err != nil {
// take-in.taken is acted on only on the runner's own takeins durable, so no step traffic holds a take-in; the main
// durable acknowledges it at once (design 47 H2 H-A-500 (b) and its addenda).
if err := dropTakenIn(heard); err != nil {
return err
}
takeIns := &oneHandler{}
takeInsName := cred.EventConsumerName() + "_" + takeInsSuffix
takeInsHeard := agedHub{hub: takeIns, ledger: process.NewConsumerLedger(process.EventsStream, takeInsName, consumersSink(consumers), nil, logger.Printf)}
if err := hearTakenIn(takeInsHeard, starter, rv.Catalog, logger.Printf); err != nil {
return err
}
if err := hearReleased(heard, hearer, starter.Registration, logger.Printf); err != nil {
@@ -393,6 +401,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// their callers against (A-186): registered before the hub starts, read again once its events are live.
seats := seatlist.NewPart("runner")
seats.Follow(seatlist.ProcessBus(calls.Call, hub), nil, logger.Printf)
bindTakeIns(ctx, js, cred, takeIns.handle, waitToBind, logger.Printf)
if err := hub.StartUntilBound(ctx, js, cred, nil, seats.EventsLive); err != nil {
logger.Printf("the consumer %s is not bound yet (%v): bound once the bus makes it", cred.EventConsumerName(), err)
}
@@ -172,6 +172,7 @@ func testTheQueueRunsPipelinesInTurn(t *testing.T) {
return ""
}, logs)
}()
takeInsHeard(t, ctx, logs)
return func() { stop(); <-done }
}
stopRunner := start()
@@ -52,6 +52,7 @@ func (s *syncBuffer) String() string { s.mu.Lock(); defer s.mu.Unlock(); return
// bus proved; the pipeline then ends passed.
func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *testing.T) {
t.Run("the queue runs pipelines in turn", testTheQueueRunsPipelinesInTurn)
t.Run("a take-in is heard while late events queue on the main durable", testATakeInIsHeardWhileLateEventsQueue)
t.Run("step log read does not wait for a batch it cannot fill", testStepLogReadDoesNotWaitForABatchItCannotFill)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
@@ -384,6 +385,7 @@ func startWhole(t *testing.T, ctx context.Context, regs []catalog.Registration,
}, logs)
}()
t.Cleanup(func() { stop(); <-done })
takeInsHeard(t, ctx, logs)
pipelines, err := js.KeyValue(ctx, module+"_pipelines")
if err != nil {
t.Fatal(err)
@@ -500,6 +502,13 @@ func findingsStream(t *testing.T, ctx context.Context, js jetstream.JetStream, m
}
}
// takeInsHeard waits until the runner's takeins durable is bound, which the runner makes at its first start from the
// moment it is made (H-A-500 (b)): a take-in said before is heard by no one, as before the runner's first start.
func takeInsHeard(t *testing.T, ctx context.Context, logs *syncBuffer) {
t.Helper()
waitFor(t, ctx, logs, func() bool { return strings.Contains(logs.String(), "take-ins: heard on the consumer") })
}
func waitFor(t *testing.T, ctx context.Context, logs *syncBuffer, cond func() bool) {
t.Helper()
for ctx.Err() == nil {
@@ -3,12 +3,17 @@ package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"time"
"github.com/nats-io/nats.go/jetstream"
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-pipeline/internal/pipeline"
)
@@ -57,6 +62,98 @@ func revisedHear(revised func(ev catalog.CatalogRevisionEvent) (int, error), log
}
}
// takeInsSuffix names the runner's own second durable on the shared events stream, `<node>_<module>_takeins`, on which alone
// take-in.taken is acted on, so a pipeline starts within seconds of its take-in whatever the step traffic on the runner's
// main durable (design 47 H2 H-A-500 (b) and its addenda).
const takeInsSuffix = "takeins"
// The ack wait and the deliveries of the takeins durable, as the grants part gives a module's consumers (H-A-500 (b)).
const (
takeInsAckWait = 30 * time.Second
takeInsMaxDeliver = 5
)
// takeInsConsumerConfig is the takeins durable as the runner makes it: take-in.taken alone, acknowledged one by one, from
// the moment it is made (H-A-500 (b)).
func takeInsConsumerConfig(cred process.Account) jetstream.ConsumerConfig {
cfg := cred.SeatEventConsumerConfig([]string{pipeline.TakenInSubject}, false)
cfg.Durable = cred.EventConsumerName() + "_" + takeInsSuffix
cfg.AckWait, cfg.MaxDeliver = takeInsAckWait, takeInsMaxDeliver
cfg.DeliverPolicy = jetstream.DeliverNewPolicy
return cfg
}
// oneHandler is the handlers of one durable that hears one pattern: the takeins durable's (H-A-500 (b)).
type oneHandler struct {
pattern string
handle func(jetstream.Msg)
}
func (o *oneHandler) Handle(pattern string, handle func(jetstream.Msg)) error {
if o.handle != nil {
return fmt.Errorf("the durable hears %q already, so %q has no taker", o.pattern, pattern)
}
o.pattern, o.handle = pattern, handle
return nil
}
// dropTakenIn has the main durable acknowledge each take-in.taken at once with no work, the takeins durable alone acting on
// it, so the finding event-unhandled stays clear (H-A-500 third addendum).
func dropTakenIn(hub handlers) error {
return hub.Handle(pipeline.TakenInSubject, func(m jetstream.Msg) { _ = m.Ack() })
}
// bindTakeIns binds the runner's takeins durable and hears it with handle, the durable made when the bus does not hold it
// (H-A-500 (b)): bound at once when it can be, so no take-in said after the runner's start is missed; a bind that fails is
// said and tried again after each wait until it binds or ctx ends, as the hub binds the main durable.
func bindTakeIns(ctx context.Context, js jetstream.JetStream, cred process.Account, handle func(jetstream.Msg), wait func(context.Context) bool,
logf func(string, ...any)) {
bind := func() error {
cons, err := cred.NamedEventConsumer(ctx, js, process.EventsStream, takeInsSuffix)
if errors.Is(err, jetstream.ErrConsumerNotFound) {
cfg := takeInsConsumerConfig(cred)
if cons, err = js.CreateConsumer(ctx, process.EventsStream, cfg); err != nil {
return fmt.Errorf("the consumer %s on %s is not there and is not made: %w", cfg.Durable, process.EventsStream, err)
}
logf("take-ins: the consumer %s on %s was not there: the runner made it (H-A-500 (b))", cfg.Durable, process.EventsStream)
}
if err != nil {
return err
}
_, err = process.HearEvents(cons, handle, logf)
return err
}
try := func() bool {
err := bind()
if err == nil {
logf("take-ins: heard on the consumer %s_%s (H-A-500 (b))", cred.EventConsumerName(), takeInsSuffix)
return true
}
logf("take-ins: the consumer %s_%s is not bound, so no take-in starts a pipeline yet; tried again: %v", cred.EventConsumerName(), takeInsSuffix, err)
return false
}
if try() {
return
}
go func() {
for wait(ctx) && !try() {
}
}()
}
// waitToBind waits five seconds or for ctx before the takeins durable is bound again: the bus makes the grant when the
// runner's account is composed, which nothing the runner hears says first (as the event hub's StartUntilBound waits).
func waitToBind(ctx context.Context) bool {
t := time.NewTimer(5 * time.Second)
defer t.Stop()
select {
case <-ctx.Done():
return false
case <-t.C:
return true
}
}
// hearTakenIn starts the pipeline of each take-in.taken heard (design 47 H §746): one per take-in, the message acknowledged
// once it is written, and heard again when a write failed, so the key already written starts nothing twice.
func hearTakenIn(hub handlers, starter *pipeline.Starter, v *pipeline.Views, logf func(string, ...any)) error {
@@ -0,0 +1,88 @@
//go:build integration
package main
import (
"context"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/testbus"
"mesh-delivery-pipeline/internal/pipeline"
)
// Design 47 H2 H-A-500 (b) and its addenda: a take-in.taken said while 500 late step-progress events wait on the runner's
// main durable, its handler held, is heard at once on the runner's own takeins durable, which the runner makes itself; the
// main durable acknowledges the take-in with no work. testATakeInIsHeardWhileLateEventsQueue is a row of
// TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview (the module's tests are folded under its budget).
func testATakeInIsHeardWhileLateEventsQueue(t *testing.T) {
url := testbus.URL(t)
nc, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
defer nc.Close()
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
cred := process.Account{URL: url, Node: "n1", Module: "mesh-delivery-pipeline"}
events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}})
if err != nil {
t.Fatal(err)
}
if _, err := events.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{Durable: cred.EventConsumerName(), AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
t.Fatal(err)
}
const progress = "mesh.seat.node-build-go.event.step-progress.n1"
for i := 0; i < 500; i++ {
if _, err := js.Publish(ctx, progress, []byte(`{"ask":"late","phase":"run"}`)); err != nil {
t.Fatal(err)
}
}
// The main durable's step-progress handler is held until the test ends: every late event waits behind the first.
held := make(chan struct{})
defer close(held)
hub := process.NewEventHub(t.Logf)
if err := hub.Handle(progress, func(m jetstream.Msg) { <-held; _ = m.Ack() }); err != nil {
t.Fatal(err)
}
if err := dropTakenIn(hub); err != nil {
t.Fatal(err)
}
if err := hub.Start(ctx, js, cred); err != nil {
t.Fatal(err)
}
defer hub.Stop()
heard := make(chan string, 1)
takeIns := &oneHandler{}
if err := takeIns.Handle(pipeline.TakenInSubject, func(m jetstream.Msg) { heard <- m.Subject(); _ = m.Ack() }); err != nil {
t.Fatal(err)
}
bindTakeIns(ctx, js, cred, takeIns.handle, func(context.Context) bool { return false }, t.Logf)
if _, err := js.Publish(ctx, "mesh.mod.mesh-delivery-indexer.event.take-in.taken", []byte(`{}`)); err != nil {
t.Fatal(err)
}
select {
case s := <-heard:
if s != "mesh.mod.mesh-delivery-indexer.event.take-in.taken" {
t.Errorf("the takeins durable heard %s", s)
}
case <-ctx.Done():
t.Fatal("the take-in is not heard while late events wait on the main durable")
}
info, err := events.Consumer(ctx, cred.EventConsumerName()+"_"+takeInsSuffix)
if err != nil {
t.Fatalf("the takeins durable is not made: %v", err)
}
if f := info.CachedInfo().Config.FilterSubjects; len(f) != 1 || f[0] != pipeline.TakenInSubject {
t.Errorf("the takeins durable filters %v, want take-in.taken alone", f)
}
}
@@ -75,7 +75,10 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(module, seat.StateStepLog), Subjects: []string{buscompose.LogSubject(module, seat.StateStepLog) + ".>"}}); err != nil {
t.Fatal(err)
}
events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>", "mesh.stream.*.event.>"}})
// The runner's findings ride its own stream, as its manifest declares, never EVENTS: on EVENTS each one would reach its
// main durable unhandled and write event-unhandled again, a flood a take-in on its own durable overtakes (H-A-500 (b)).
findingsStream(t, ctx, js, module)
events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}})
if err != nil {
t.Fatal(err)
}
@@ -224,6 +227,7 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
runCtx, stop := context.WithCancel(ctx)
done := make(chan error, 1)
go func() { done <- run(runCtx, func(k string) string { return env[k] }, logs) }()
takeInsHeard(t, ctx, logs)
return func() { stop(); <-done }
}
stop := start()