Runner: a take-in said before the takeins durable is bound is still worked

The takeins durable delivers from the moment it is made, so a take-in said
while the runner was down reached only the main durable, which acknowledged
it with no work: that take-in started no pipeline. Until the takeins durable
is bound, the main durable now works each take-in.taken; from the bind, it
acknowledges with no work those the takeins durable delivers, told apart by
the stream sequence the durable delivers after, so a take-in still waiting on
the main durable is worked whatever the moment it is heard.

Builds: design 47 H2 H-A-500 fourth addendum
This commit is contained in:
2026-11-22 22:59:21 +01:00
parent b2d9d1ac90
commit 678d8fa5b3
5 changed files with 197 additions and 18 deletions
@@ -60,3 +60,37 @@ func TestAgedHub(t *testing.T) {
t.Errorf("a message asked again is noted applied: %+v", last)
}
}
// TestTheTakeInsSwitch: until the takeins durable is bound, the main durable works every take-in; from the bind, it works
// only those said at or before the sequence the durable delivers after, and acknowledges the rest with no work
// (design 47 H2 H-A-500 fourth addendum).
func TestTheTakeInsSwitch(t *testing.T) {
for _, row := range []struct {
name string
boundFrom uint64
bound bool
seq uint64
delivers bool
}{
{"not bound, any sequence is worked", 0, false, 7, false},
{"bound after 5, the take-in at 5 is worked", 5, true, 5, false},
{"bound after 5, the take-in at 6 is the durable's", 5, true, 6, true},
{"bound on an empty stream, the first take-in is the durable's", 0, true, 1, true},
} {
t.Run(row.name, func(t *testing.T) {
sw := &takeInsSwitch{}
if row.bound {
sw.boundFrom(row.boundFrom)
}
hub := &oneHub{}
worked := false
if err := dropTakenIn(hub, sw, func(jetstream.Msg) { worked = true }); err != nil {
t.Fatal(err)
}
hub.handle(ackedMsg{seq: row.seq})
if worked == row.delivers {
t.Errorf("worked by the main durable: %v, want %v", worked, !row.delivers)
}
})
}
}
@@ -376,9 +376,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err := hearRecovery(heard, kvRecoveries{ctx, recovered}, logger.Printf); err != nil {
return err
}
// take-in.taken is acted on only on the runner's own takeins durable, so no step traffic holds a take-in; the main
// durable acknowledges it at once (design 47 H2 H-A-500 (b) and its addenda).
if err := dropTakenIn(heard); err != nil {
// take-in.taken is acted on on the runner's own takeins durable once that is bound, so no step traffic holds a take-in;
// until then the main durable works it, and from the bind acknowledges it at once (design 47 H2 H-A-500 (b) and its
// addenda, the fourth: no take-in said while the runner was down is lost).
takeInsBound := &takeInsSwitch{}
if err := dropTakenIn(heard, takeInsBound, takenIn(starter, rv.Catalog, logger.Printf)); err != nil {
return err
}
takeIns := &oneHandler{}
@@ -401,7 +403,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// their callers against (A-186): registered before the hub starts, read again once its events are live.
seats := seatlist.NewPart("runner")
seats.Follow(seatlist.ProcessBus(calls.Call, hub), nil, logger.Printf)
bindTakeIns(ctx, js, cred, takeIns.handle, waitToBind, logger.Printf)
bindTakeIns(ctx, js, cred, takeIns.handle, takeInsBound, waitToBind, logger.Printf)
if err := hub.StartUntilBound(ctx, js, cred, nil, seats.EventsLive); err != nil {
logger.Printf("the consumer %s is not bound yet (%v): bound once the bus makes it", cred.EventConsumerName(), err)
}
@@ -53,6 +53,7 @@ func (s *syncBuffer) String() string { s.mu.Lock(); defer s.mu.Unlock(); return
func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *testing.T) {
t.Run("the queue runs pipelines in turn", testTheQueueRunsPipelinesInTurn)
t.Run("a take-in is heard while late events queue on the main durable", testATakeInIsHeardWhileLateEventsQueue)
t.Run("a take-in said before the bind is still worked", testATakeInSaidBeforeTheBindIsStillWorked)
t.Run("step log read does not wait for a batch it cannot fill", testStepLogReadDoesNotWaitForABatchItCannotFill)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
@@ -503,7 +504,8 @@ func findingsStream(t *testing.T, ctx context.Context, js jetstream.JetStream, m
}
// takeInsHeard waits until the runner's takeins durable is bound, which the runner makes at its first start from the
// moment it is made (H-A-500 (b)): a take-in said before is heard by no one, as before the runner's first start.
// moment it is made (H-A-500 (b)); a take-in said before is worked by the main durable (the fourth addendum), but the tests
// publish after the bind so that each take-in is heard where the running runner hears it.
func takeInsHeard(t *testing.T, ctx context.Context, logs *syncBuffer) {
t.Helper()
waitFor(t, ctx, logs, func() bool { return strings.Contains(logs.String(), "take-ins: heard on the consumer") })
@@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
"sync/atomic"
"time"
"github.com/nats-io/nats.go/jetstream"
@@ -97,17 +98,52 @@ func (o *oneHandler) Handle(pattern string, handle func(jetstream.Msg)) error {
return nil
}
// dropTakenIn has the main durable acknowledge each take-in.taken at once with no work, the takeins durable alone acting on
// it, so the finding event-unhandled stays clear (H-A-500 third addendum).
func dropTakenIn(hub handlers) error {
return hub.Handle(pipeline.TakenInSubject, func(m jetstream.Msg) { _ = m.Ack() })
// takeInsSwitch says which durable a take-in.taken is worked on (H-A-500 fourth addendum): until the runner's takeins
// durable is bound, the main durable does the work, so a take-in said while the runner was down is lost to no one; from
// the bind, the takeins durable alone works the take-ins said after it was made, and the main durable acknowledges those
// at once with no work. The boundary is the stream sequence the takeins durable delivers from, so a take-in said before
// the durable was made and still waiting on the main durable is worked there whatever the moment it is heard.
type takeInsSwitch struct {
// from holds 1 + the stream sequence the takeins durable delivers after; 0 while it is not bound.
from atomic.Uint64
}
// boundFrom says the takeins durable is bound and delivers every take-in said after stream sequence seq.
func (s *takeInsSwitch) boundFrom(seq uint64) { s.from.Store(seq + 1) }
// delivers says whether the takeins durable delivers m, so the main durable has no work on it.
func (s *takeInsSwitch) delivers(m jetstream.Msg) bool {
from := s.from.Load()
if from == 0 {
return false
}
meta, err := m.Metadata()
if err != nil {
// A message whose sequence is unknown is worked rather than dropped: the work is done once per key, never twice.
return false
}
return meta.Sequence.Stream >= from
}
// dropTakenIn has the main durable work each take-in.taken the takeins durable does not deliver, and acknowledge the rest at
// once with no work, so the finding event-unhandled stays clear (H-A-500 third and fourth addenda).
func dropTakenIn(hub handlers, sw *takeInsSwitch, work func(jetstream.Msg)) error {
return hub.Handle(pipeline.TakenInSubject, func(m jetstream.Msg) {
if sw.delivers(m) {
_ = m.Ack()
return
}
work(m)
})
}
// bindTakeIns binds the runner's takeins durable and hears it with handle, the durable made when the bus does not hold it
// (H-A-500 (b)): bound at once when it can be, so no take-in said after the runner's start is missed; a bind that fails is
// said and tried again after each wait until it binds or ctx ends, as the hub binds the main durable.
func bindTakeIns(ctx context.Context, js jetstream.JetStream, cred process.Account, handle func(jetstream.Msg), wait func(context.Context) bool,
logf func(string, ...any)) {
// said and tried again after each wait until it binds or ctx ends, as the hub binds the main durable. Once bound, sw says
// so with the stream sequence the durable delivers after, and the main durable stops working the take-ins it delivers
// (H-A-500 fourth addendum).
func bindTakeIns(ctx context.Context, js jetstream.JetStream, cred process.Account, handle func(jetstream.Msg), sw *takeInsSwitch,
wait func(context.Context) bool, logf func(string, ...any)) {
bind := func() error {
cons, err := cred.NamedEventConsumer(ctx, js, process.EventsStream, takeInsSuffix)
if errors.Is(err, jetstream.ErrConsumerNotFound) {
@@ -120,8 +156,16 @@ func bindTakeIns(ctx context.Context, js jetstream.JetStream, cred process.Accou
if err != nil {
return err
}
_, err = process.HearEvents(cons, handle, logf)
return err
// A durable just made delivers after the stream's last sequence; one found, after what it delivered already.
info, err := cons.Info(ctx)
if err != nil {
return fmt.Errorf("the consumer %s_%s is not read: %w", cred.EventConsumerName(), takeInsSuffix, err)
}
if _, err = process.HearEvents(cons, handle, logf); err != nil {
return err
}
sw.boundFrom(info.Delivered.Stream)
return nil
}
try := func() bool {
err := bind()
@@ -157,7 +201,12 @@ func waitToBind(ctx context.Context) bool {
// hearTakenIn starts the pipeline of each take-in.taken heard (design 47 H §746): one per take-in, the message acknowledged
// once it is written, and heard again when a write failed, so the key already written starts nothing twice.
func hearTakenIn(hub handlers, starter *pipeline.Starter, v *pipeline.Views, logf func(string, ...any)) error {
return hub.Handle(pipeline.TakenInSubject, func(m jetstream.Msg) {
return hub.Handle(pipeline.TakenInSubject, takenIn(starter, v, logf))
}
// takenIn is the work of one take-in.taken, on whichever durable delivers it (H-A-500 fourth addendum).
func takenIn(starter *pipeline.Starter, v *pipeline.Views, logf func(string, ...any)) func(jetstream.Msg) {
return func(m jetstream.Msg) {
// An event of the old form is refused by the contract's strict decode, said in one journal line and ended (§765, H-A-98).
ev, err := pipeline.DecodeTakenIn(m.Data())
if err != nil {
@@ -173,7 +222,7 @@ func hearTakenIn(hub handlers, starter *pipeline.Starter, v *pipeline.Views, log
}
logf("the take-in of %s source %d started %d pipelines", ev.Branch, ev.Source, n)
_ = m.Ack()
})
}
}
// hearReleased writes each version-released's version into its mover (design 47 H §746, H-A-52), so the pipeline's end names
@@ -54,7 +54,8 @@ func testATakeInIsHeardWhileLateEventsQueue(t *testing.T) {
if err := hub.Handle(progress, func(m jetstream.Msg) { <-held; _ = m.Ack() }); err != nil {
t.Fatal(err)
}
if err := dropTakenIn(hub); err != nil {
sw := &takeInsSwitch{}
if err := dropTakenIn(hub, sw, func(m jetstream.Msg) { t.Errorf("the main durable worked %s after the bind", m.Subject()); _ = m.Ack() }); err != nil {
t.Fatal(err)
}
if err := hub.Start(ctx, js, cred); err != nil {
@@ -66,7 +67,7 @@ func testATakeInIsHeardWhileLateEventsQueue(t *testing.T) {
if err := takeIns.Handle(pipeline.TakenInSubject, func(m jetstream.Msg) { heard <- m.Subject(); _ = m.Ack() }); err != nil {
t.Fatal(err)
}
bindTakeIns(ctx, js, cred, takeIns.handle, func(context.Context) bool { return false }, t.Logf)
bindTakeIns(ctx, js, cred, takeIns.handle, sw, func(context.Context) bool { return false }, t.Logf)
if _, err := js.Publish(ctx, "mesh.mod.mesh-delivery-indexer.event.take-in.taken", []byte(`{}`)); err != nil {
t.Fatal(err)
}
@@ -86,3 +87,94 @@ func testATakeInIsHeardWhileLateEventsQueue(t *testing.T) {
t.Errorf("the takeins durable filters %v, want take-in.taken alone", f)
}
}
// Design 47 H2 H-A-500 fourth addendum: a take-in.taken said before the runner's takeins durable is bound (the runner was
// down, or the bind failed) is worked by the main durable, so it starts its pipeline; one said after the bind is worked by
// the takeins durable alone and the main durable acknowledges it with no work. A row of
// TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview.
func testATakeInSaidBeforeTheBindIsStillWorked(t *testing.T) {
url := testbus.URL(t)
nc, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
defer nc.Close()
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
cred := process.Account{URL: url, Node: "n1", Module: "mesh-delivery-pipeline"}
events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}})
if err != nil {
t.Fatal(err)
}
if _, err := events.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{Durable: cred.EventConsumerName(), AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
t.Fatal(err)
}
const subject = "mesh.mod.mesh-delivery-indexer.event.take-in.taken"
// Said while the runner was down: the main durable holds it, the takeins durable does not exist yet.
if _, err := js.Publish(ctx, subject, []byte(`{"before":true}`)); err != nil {
t.Fatal(err)
}
// The main durable's take-in handler is held until the takeins durable is bound, as a main durable busy with step
// events would be: the take-in said before the bind is still worked, by its sequence, not by the moment it is heard.
held := make(chan struct{})
worked, heard := make(chan string, 2), make(chan string, 2)
hub := process.NewEventHub(t.Logf)
sw := &takeInsSwitch{}
if err := dropTakenIn(hub, sw, func(m jetstream.Msg) { <-held; worked <- string(m.Data()); _ = m.Ack() }); err != nil {
t.Fatal(err)
}
if err := hub.Start(ctx, js, cred); err != nil {
t.Fatal(err)
}
defer hub.Stop()
takeIns := &oneHandler{}
if err := takeIns.Handle(pipeline.TakenInSubject, func(m jetstream.Msg) { heard <- string(m.Data()); _ = m.Ack() }); err != nil {
t.Fatal(err)
}
bindTakeIns(ctx, js, cred, takeIns.handle, sw, func(context.Context) bool { return false }, t.Logf)
if sw.from.Load() == 0 {
t.Fatal("the takeins durable is not bound")
}
// Said after the bind: the takeins durable's alone.
if _, err := js.Publish(ctx, subject, []byte(`{"after":true}`)); err != nil {
t.Fatal(err)
}
close(held)
select {
case d := <-worked:
if d != `{"before":true}` {
t.Errorf("the main durable worked %s, want the take-in said before the bind", d)
}
case <-ctx.Done():
t.Fatal("the take-in said before the bind is worked by no durable")
}
select {
case d := <-heard:
if d != `{"after":true}` {
t.Errorf("the takeins durable heard %s, want the take-in said after the bind", d)
}
case <-ctx.Done():
t.Fatal("the take-in said after the bind is not heard on the takeins durable")
}
// The main durable acknowledged the second with no work: nothing more is worked once its queue is drained.
select {
case d := <-worked:
t.Errorf("the main durable worked %s after the bind", d)
case <-time.After(2 * time.Second):
}
info, err := events.Consumer(ctx, cred.EventConsumerName())
if err != nil {
t.Fatal(err)
}
ci, err := info.Info(ctx)
if err != nil {
t.Fatal(err)
}
if ci.NumAckPending != 0 || ci.NumPending != 0 {
t.Errorf("the main durable holds %d pending and %d unacknowledged, want none", ci.NumPending, ci.NumAckPending)
}
}