Runner: a takeins bind that failed is bound again on registration-changed, not on a timer

A takeins bind that failed was tried again every five seconds. Nothing in the
mesh waits on a timer for another part: the runner binds its takeins durable
at its start and again on each registration-changed, as the indexer binds a
branch's consumer, and says each failed bind; until bound, the main durable
works each take-in.

Builds: design 47 H2 H-A-500 fifth addendum
This commit is contained in:
2026-11-22 23:03:41 +01:00
parent 5792e83bc6
commit a8240f5549
4 changed files with 74 additions and 35 deletions
@@ -237,8 +237,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
}
advancer.Store(&advance)
// A registration-changed places the steps of its module's pipelines started before it, from it (A-327).
// A registration-changed places the steps of its module's pipelines started before it, from it (A-327), and binds the
// takeins durable when its bind failed at the start (H-A-500 fifth addendum).
var bindTakeInsAgain atomic.Pointer[func()]
register := func(reg catalog.Registration) {
if f := bindTakeInsAgain.Load(); f != nil {
(*f)()
}
if n, err := starter.Registered(store, reg, rv.Catalog.View); err != nil || n > 0 {
logger.Printf("the pipelines of %s started before its registration: %d placed (%v)", reg.Module, n, err)
}
@@ -403,7 +408,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// their callers against (A-186): registered before the hub starts, read again once its events are live.
seats := seatlist.NewPart("runner")
seats.Follow(seatlist.ProcessBus(calls.Call, hub), nil, logger.Printf)
bindTakeIns(ctx, js, cred, takeIns.handle, takeInsBound, waitToBind, logger.Printf)
again := bindTakeIns(ctx, js, cred, takeIns.handle, takeInsBound, logger.Printf)
bindTakeInsAgain.Store(&again)
if err := hub.StartUntilBound(ctx, js, cred, nil, seats.EventsLive); err != nil {
logger.Printf("the consumer %s is not bound yet (%v): bound once the bus makes it", cred.EventConsumerName(), err)
}
@@ -54,6 +54,7 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes
t.Run("the queue runs pipelines in turn", testTheQueueRunsPipelinesInTurn)
t.Run("a take-in is heard while late events queue on the main durable", testATakeInIsHeardWhileLateEventsQueue)
t.Run("a take-in said before the bind is still worked", testATakeInSaidBeforeTheBindIsStillWorked)
t.Run("a takeins bind that failed is bound again on registration-changed", testATakeInsBindThatFailedIsBoundAgainOnRegistrationChanged)
t.Run("step log read does not wait for a batch it cannot fill", testStepLogReadDoesNotWaitForABatchItCannotFill)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
@@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
"sync"
"sync/atomic"
"time"
@@ -138,12 +139,13 @@ func dropTakenIn(hub handlers, sw *takeInsSwitch, work func(jetstream.Msg)) erro
}
// bindTakeIns binds the runner's takeins durable and hears it with handle, the durable made when the bus does not hold it
// (H-A-500 (b)): bound at once when it can be, so no take-in said after the runner's start is missed; a bind that fails is
// said and tried again after each wait until it binds or ctx ends, as the hub binds the main durable. Once bound, sw says
// so with the stream sequence the durable delivers after, and the main durable stops working the take-ins it delivers
// (H-A-500 fourth addendum).
// (H-A-500 (b)): bound at once when it can be, so no take-in said after the runner's start is missed. A bind that fails
// (the bus has not made the runner's grant yet) is said, and the answered func binds again, called on each
// registration-changed as the indexer binds a branch's consumer on each (H-A-500 fifth addendum: nothing waits on a timer
// for another part); once bound it does nothing. Once bound, sw says so with the stream sequence the durable delivers
// after, and the main durable stops working the take-ins it delivers (the fourth addendum).
func bindTakeIns(ctx context.Context, js jetstream.JetStream, cred process.Account, handle func(jetstream.Msg), sw *takeInsSwitch,
wait func(context.Context) bool, logf func(string, ...any)) {
logf func(string, ...any)) (again func()) {
bind := func() error {
cons, err := cred.NamedEventConsumer(ctx, js, process.EventsStream, takeInsSuffix)
if errors.Is(err, jetstream.ErrConsumerNotFound) {
@@ -167,35 +169,24 @@ func bindTakeIns(ctx context.Context, js jetstream.JetStream, cred process.Accou
sw.boundFrom(info.Delivered.Stream)
return nil
}
try := func() bool {
err := bind()
if err == nil {
logf("take-ins: heard on the consumer %s_%s (H-A-500 (b))", cred.EventConsumerName(), takeInsSuffix)
return true
var mu sync.Mutex
bound := false
try := func() {
mu.Lock()
defer mu.Unlock()
if bound || ctx.Err() != nil {
return
}
logf("take-ins: the consumer %s_%s is not bound, so no take-in starts a pipeline yet; tried again: %v", cred.EventConsumerName(), takeInsSuffix, err)
return false
}
if try() {
return
}
go func() {
for wait(ctx) && !try() {
if err := bind(); err != nil {
logf("take-ins: the consumer %s_%s is not bound, so the main durable works each take-in; bound again at the next registration-changed: %v",
cred.EventConsumerName(), takeInsSuffix, err)
return
}
}()
}
// waitToBind waits five seconds or for ctx before the takeins durable is bound again: the bus makes the grant when the
// runner's account is composed, which nothing the runner hears says first (as the event hub's StartUntilBound waits).
func waitToBind(ctx context.Context) bool {
t := time.NewTimer(5 * time.Second)
defer t.Stop()
select {
case <-ctx.Done():
return false
case <-t.C:
return true
bound = true
logf("take-ins: heard on the consumer %s_%s (H-A-500 (b))", cred.EventConsumerName(), takeInsSuffix)
}
try()
return try
}
// hearTakenIn starts the pipeline of each take-in.taken heard (design 47 H §746): one per take-in, the message acknowledged
@@ -67,7 +67,7 @@ func testATakeInIsHeardWhileLateEventsQueue(t *testing.T) {
if err := takeIns.Handle(pipeline.TakenInSubject, func(m jetstream.Msg) { heard <- m.Subject(); _ = m.Ack() }); err != nil {
t.Fatal(err)
}
bindTakeIns(ctx, js, cred, takeIns.handle, sw, func(context.Context) bool { return false }, t.Logf)
bindTakeIns(ctx, js, cred, takeIns.handle, sw, t.Logf)
if _, err := js.Publish(ctx, "mesh.mod.mesh-delivery-indexer.event.take-in.taken", []byte(`{}`)); err != nil {
t.Fatal(err)
}
@@ -135,7 +135,7 @@ func testATakeInSaidBeforeTheBindIsStillWorked(t *testing.T) {
if err := takeIns.Handle(pipeline.TakenInSubject, func(m jetstream.Msg) { heard <- string(m.Data()); _ = m.Ack() }); err != nil {
t.Fatal(err)
}
bindTakeIns(ctx, js, cred, takeIns.handle, sw, func(context.Context) bool { return false }, t.Logf)
bindTakeIns(ctx, js, cred, takeIns.handle, sw, t.Logf)
if sw.from.Load() == 0 {
t.Fatal("the takeins durable is not bound")
}
@@ -178,3 +178,44 @@ func testATakeInSaidBeforeTheBindIsStillWorked(t *testing.T) {
t.Errorf("the main durable holds %d pending and %d unacknowledged, want none", ci.NumPending, ci.NumAckPending)
}
}
// Design 47 H2 H-A-500 fifth addendum: a takeins bind that failed at the start (here the events stream is not there yet, as
// the runner's grant is not there before the bus composes its account) is said and bound again on the next
// registration-changed, never on a timer. A row of TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview.
func testATakeInsBindThatFailedIsBoundAgainOnRegistrationChanged(t *testing.T) {
url := testbus.URL(t)
nc, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
defer nc.Close()
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
cred := process.Account{URL: url, Node: "n1", Module: "mesh-delivery-pipeline"}
sw := &takeInsSwitch{}
heard := make(chan string, 1)
again := bindTakeIns(ctx, js, cred, func(m jetstream.Msg) { heard <- m.Subject(); _ = m.Ack() }, sw, t.Logf)
if sw.from.Load() != 0 {
t.Fatal("the takeins durable is bound with no events stream")
}
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}}); err != nil {
t.Fatal(err)
}
again() // the registration-changed heard once the bus holds the stream and the grant
if sw.from.Load() == 0 {
t.Fatal("the takeins durable is not bound again on registration-changed")
}
if _, err := js.Publish(ctx, "mesh.mod.mesh-delivery-indexer.event.take-in.taken", []byte(`{}`)); err != nil {
t.Fatal(err)
}
select {
case <-heard:
case <-ctx.Done():
t.Fatal("the take-in is not heard on the durable bound again")
}
again() // bound already: nothing is bound twice
}