Runner: a take-in said before the takeins durable is bound is still worked
The takeins durable delivers from the moment it is made, so a take-in said while the runner was down reached only the main durable, which acknowledged it with no work: that take-in started no pipeline. Until the takeins durable is bound, the main durable now works each take-in.taken; from the bind, it acknowledges with no work those the takeins durable delivers, told apart by the stream sequence the durable delivers after, so a take-in still waiting on the main durable is worked whatever the moment it is heard. Builds: design 47 H2 H-A-500 fourth addendum
This commit is contained in:
@@ -60,3 +60,37 @@ func TestAgedHub(t *testing.T) {
|
||||
t.Errorf("a message asked again is noted applied: %+v", last)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTheTakeInsSwitch: until the takeins durable is bound, the main durable works every take-in; from the bind, it works
|
||||
// only those said at or before the sequence the durable delivers after, and acknowledges the rest with no work
|
||||
// (design 47 H2 H-A-500 fourth addendum).
|
||||
func TestTheTakeInsSwitch(t *testing.T) {
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
boundFrom uint64
|
||||
bound bool
|
||||
seq uint64
|
||||
delivers bool
|
||||
}{
|
||||
{"not bound, any sequence is worked", 0, false, 7, false},
|
||||
{"bound after 5, the take-in at 5 is worked", 5, true, 5, false},
|
||||
{"bound after 5, the take-in at 6 is the durable's", 5, true, 6, true},
|
||||
{"bound on an empty stream, the first take-in is the durable's", 0, true, 1, true},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
sw := &takeInsSwitch{}
|
||||
if row.bound {
|
||||
sw.boundFrom(row.boundFrom)
|
||||
}
|
||||
hub := &oneHub{}
|
||||
worked := false
|
||||
if err := dropTakenIn(hub, sw, func(jetstream.Msg) { worked = true }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hub.handle(ackedMsg{seq: row.seq})
|
||||
if worked == row.delivers {
|
||||
t.Errorf("worked by the main durable: %v, want %v", worked, !row.delivers)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -376,9 +376,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err := hearRecovery(heard, kvRecoveries{ctx, recovered}, logger.Printf); err != nil {
|
||||
return err
|
||||
}
|
||||
// take-in.taken is acted on only on the runner's own takeins durable, so no step traffic holds a take-in; the main
|
||||
// durable acknowledges it at once (design 47 H2 H-A-500 (b) and its addenda).
|
||||
if err := dropTakenIn(heard); err != nil {
|
||||
// take-in.taken is acted on on the runner's own takeins durable once that is bound, so no step traffic holds a take-in;
|
||||
// until then the main durable works it, and from the bind acknowledges it at once (design 47 H2 H-A-500 (b) and its
|
||||
// addenda, the fourth: no take-in said while the runner was down is lost).
|
||||
takeInsBound := &takeInsSwitch{}
|
||||
if err := dropTakenIn(heard, takeInsBound, takenIn(starter, rv.Catalog, logger.Printf)); err != nil {
|
||||
return err
|
||||
}
|
||||
takeIns := &oneHandler{}
|
||||
@@ -401,7 +403,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// their callers against (A-186): registered before the hub starts, read again once its events are live.
|
||||
seats := seatlist.NewPart("runner")
|
||||
seats.Follow(seatlist.ProcessBus(calls.Call, hub), nil, logger.Printf)
|
||||
bindTakeIns(ctx, js, cred, takeIns.handle, waitToBind, logger.Printf)
|
||||
bindTakeIns(ctx, js, cred, takeIns.handle, takeInsBound, waitToBind, logger.Printf)
|
||||
if err := hub.StartUntilBound(ctx, js, cred, nil, seats.EventsLive); err != nil {
|
||||
logger.Printf("the consumer %s is not bound yet (%v): bound once the bus makes it", cred.EventConsumerName(), err)
|
||||
}
|
||||
|
||||
@@ -53,6 +53,7 @@ func (s *syncBuffer) String() string { s.mu.Lock(); defer s.mu.Unlock(); return
|
||||
func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *testing.T) {
|
||||
t.Run("the queue runs pipelines in turn", testTheQueueRunsPipelinesInTurn)
|
||||
t.Run("a take-in is heard while late events queue on the main durable", testATakeInIsHeardWhileLateEventsQueue)
|
||||
t.Run("a take-in said before the bind is still worked", testATakeInSaidBeforeTheBindIsStillWorked)
|
||||
t.Run("step log read does not wait for a batch it cannot fill", testStepLogReadDoesNotWaitForABatchItCannotFill)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
@@ -503,7 +504,8 @@ func findingsStream(t *testing.T, ctx context.Context, js jetstream.JetStream, m
|
||||
}
|
||||
|
||||
// takeInsHeard waits until the runner's takeins durable is bound, which the runner makes at its first start from the
|
||||
// moment it is made (H-A-500 (b)): a take-in said before is heard by no one, as before the runner's first start.
|
||||
// moment it is made (H-A-500 (b)); a take-in said before is worked by the main durable (the fourth addendum), but the tests
|
||||
// publish after the bind so that each take-in is heard where the running runner hears it.
|
||||
func takeInsHeard(t *testing.T, ctx context.Context, logs *syncBuffer) {
|
||||
t.Helper()
|
||||
waitFor(t, ctx, logs, func() bool { return strings.Contains(logs.String(), "take-ins: heard on the consumer") })
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
@@ -97,17 +98,52 @@ func (o *oneHandler) Handle(pattern string, handle func(jetstream.Msg)) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// dropTakenIn has the main durable acknowledge each take-in.taken at once with no work, the takeins durable alone acting on
|
||||
// it, so the finding event-unhandled stays clear (H-A-500 third addendum).
|
||||
func dropTakenIn(hub handlers) error {
|
||||
return hub.Handle(pipeline.TakenInSubject, func(m jetstream.Msg) { _ = m.Ack() })
|
||||
// takeInsSwitch says which durable a take-in.taken is worked on (H-A-500 fourth addendum): until the runner's takeins
|
||||
// durable is bound, the main durable does the work, so a take-in said while the runner was down is lost to no one; from
|
||||
// the bind, the takeins durable alone works the take-ins said after it was made, and the main durable acknowledges those
|
||||
// at once with no work. The boundary is the stream sequence the takeins durable delivers from, so a take-in said before
|
||||
// the durable was made and still waiting on the main durable is worked there whatever the moment it is heard.
|
||||
type takeInsSwitch struct {
|
||||
// from holds 1 + the stream sequence the takeins durable delivers after; 0 while it is not bound.
|
||||
from atomic.Uint64
|
||||
}
|
||||
|
||||
// boundFrom says the takeins durable is bound and delivers every take-in said after stream sequence seq.
|
||||
func (s *takeInsSwitch) boundFrom(seq uint64) { s.from.Store(seq + 1) }
|
||||
|
||||
// delivers says whether the takeins durable delivers m, so the main durable has no work on it.
|
||||
func (s *takeInsSwitch) delivers(m jetstream.Msg) bool {
|
||||
from := s.from.Load()
|
||||
if from == 0 {
|
||||
return false
|
||||
}
|
||||
meta, err := m.Metadata()
|
||||
if err != nil {
|
||||
// A message whose sequence is unknown is worked rather than dropped: the work is done once per key, never twice.
|
||||
return false
|
||||
}
|
||||
return meta.Sequence.Stream >= from
|
||||
}
|
||||
|
||||
// dropTakenIn has the main durable work each take-in.taken the takeins durable does not deliver, and acknowledge the rest at
|
||||
// once with no work, so the finding event-unhandled stays clear (H-A-500 third and fourth addenda).
|
||||
func dropTakenIn(hub handlers, sw *takeInsSwitch, work func(jetstream.Msg)) error {
|
||||
return hub.Handle(pipeline.TakenInSubject, func(m jetstream.Msg) {
|
||||
if sw.delivers(m) {
|
||||
_ = m.Ack()
|
||||
return
|
||||
}
|
||||
work(m)
|
||||
})
|
||||
}
|
||||
|
||||
// bindTakeIns binds the runner's takeins durable and hears it with handle, the durable made when the bus does not hold it
|
||||
// (H-A-500 (b)): bound at once when it can be, so no take-in said after the runner's start is missed; a bind that fails is
|
||||
// said and tried again after each wait until it binds or ctx ends, as the hub binds the main durable.
|
||||
func bindTakeIns(ctx context.Context, js jetstream.JetStream, cred process.Account, handle func(jetstream.Msg), wait func(context.Context) bool,
|
||||
logf func(string, ...any)) {
|
||||
// said and tried again after each wait until it binds or ctx ends, as the hub binds the main durable. Once bound, sw says
|
||||
// so with the stream sequence the durable delivers after, and the main durable stops working the take-ins it delivers
|
||||
// (H-A-500 fourth addendum).
|
||||
func bindTakeIns(ctx context.Context, js jetstream.JetStream, cred process.Account, handle func(jetstream.Msg), sw *takeInsSwitch,
|
||||
wait func(context.Context) bool, logf func(string, ...any)) {
|
||||
bind := func() error {
|
||||
cons, err := cred.NamedEventConsumer(ctx, js, process.EventsStream, takeInsSuffix)
|
||||
if errors.Is(err, jetstream.ErrConsumerNotFound) {
|
||||
@@ -120,8 +156,16 @@ func bindTakeIns(ctx context.Context, js jetstream.JetStream, cred process.Accou
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = process.HearEvents(cons, handle, logf)
|
||||
return err
|
||||
// A durable just made delivers after the stream's last sequence; one found, after what it delivered already.
|
||||
info, err := cons.Info(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the consumer %s_%s is not read: %w", cred.EventConsumerName(), takeInsSuffix, err)
|
||||
}
|
||||
if _, err = process.HearEvents(cons, handle, logf); err != nil {
|
||||
return err
|
||||
}
|
||||
sw.boundFrom(info.Delivered.Stream)
|
||||
return nil
|
||||
}
|
||||
try := func() bool {
|
||||
err := bind()
|
||||
@@ -157,7 +201,12 @@ func waitToBind(ctx context.Context) bool {
|
||||
// hearTakenIn starts the pipeline of each take-in.taken heard (design 47 H §746): one per take-in, the message acknowledged
|
||||
// once it is written, and heard again when a write failed, so the key already written starts nothing twice.
|
||||
func hearTakenIn(hub handlers, starter *pipeline.Starter, v *pipeline.Views, logf func(string, ...any)) error {
|
||||
return hub.Handle(pipeline.TakenInSubject, func(m jetstream.Msg) {
|
||||
return hub.Handle(pipeline.TakenInSubject, takenIn(starter, v, logf))
|
||||
}
|
||||
|
||||
// takenIn is the work of one take-in.taken, on whichever durable delivers it (H-A-500 fourth addendum).
|
||||
func takenIn(starter *pipeline.Starter, v *pipeline.Views, logf func(string, ...any)) func(jetstream.Msg) {
|
||||
return func(m jetstream.Msg) {
|
||||
// An event of the old form is refused by the contract's strict decode, said in one journal line and ended (§765, H-A-98).
|
||||
ev, err := pipeline.DecodeTakenIn(m.Data())
|
||||
if err != nil {
|
||||
@@ -173,7 +222,7 @@ func hearTakenIn(hub handlers, starter *pipeline.Starter, v *pipeline.Views, log
|
||||
}
|
||||
logf("the take-in of %s source %d started %d pipelines", ev.Branch, ev.Source, n)
|
||||
_ = m.Ack()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// hearReleased writes each version-released's version into its mover (design 47 H §746, H-A-52), so the pipeline's end names
|
||||
|
||||
+94
-2
@@ -54,7 +54,8 @@ func testATakeInIsHeardWhileLateEventsQueue(t *testing.T) {
|
||||
if err := hub.Handle(progress, func(m jetstream.Msg) { <-held; _ = m.Ack() }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := dropTakenIn(hub); err != nil {
|
||||
sw := &takeInsSwitch{}
|
||||
if err := dropTakenIn(hub, sw, func(m jetstream.Msg) { t.Errorf("the main durable worked %s after the bind", m.Subject()); _ = m.Ack() }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := hub.Start(ctx, js, cred); err != nil {
|
||||
@@ -66,7 +67,7 @@ func testATakeInIsHeardWhileLateEventsQueue(t *testing.T) {
|
||||
if err := takeIns.Handle(pipeline.TakenInSubject, func(m jetstream.Msg) { heard <- m.Subject(); _ = m.Ack() }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bindTakeIns(ctx, js, cred, takeIns.handle, func(context.Context) bool { return false }, t.Logf)
|
||||
bindTakeIns(ctx, js, cred, takeIns.handle, sw, func(context.Context) bool { return false }, t.Logf)
|
||||
if _, err := js.Publish(ctx, "mesh.mod.mesh-delivery-indexer.event.take-in.taken", []byte(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -86,3 +87,94 @@ func testATakeInIsHeardWhileLateEventsQueue(t *testing.T) {
|
||||
t.Errorf("the takeins durable filters %v, want take-in.taken alone", f)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H2 H-A-500 fourth addendum: a take-in.taken said before the runner's takeins durable is bound (the runner was
|
||||
// down, or the bind failed) is worked by the main durable, so it starts its pipeline; one said after the bind is worked by
|
||||
// the takeins durable alone and the main durable acknowledges it with no work. A row of
|
||||
// TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview.
|
||||
func testATakeInSaidBeforeTheBindIsStillWorked(t *testing.T) {
|
||||
url := testbus.URL(t)
|
||||
nc, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer nc.Close()
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
cred := process.Account{URL: url, Node: "n1", Module: "mesh-delivery-pipeline"}
|
||||
events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := events.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{Durable: cred.EventConsumerName(), AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const subject = "mesh.mod.mesh-delivery-indexer.event.take-in.taken"
|
||||
// Said while the runner was down: the main durable holds it, the takeins durable does not exist yet.
|
||||
if _, err := js.Publish(ctx, subject, []byte(`{"before":true}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The main durable's take-in handler is held until the takeins durable is bound, as a main durable busy with step
|
||||
// events would be: the take-in said before the bind is still worked, by its sequence, not by the moment it is heard.
|
||||
held := make(chan struct{})
|
||||
worked, heard := make(chan string, 2), make(chan string, 2)
|
||||
hub := process.NewEventHub(t.Logf)
|
||||
sw := &takeInsSwitch{}
|
||||
if err := dropTakenIn(hub, sw, func(m jetstream.Msg) { <-held; worked <- string(m.Data()); _ = m.Ack() }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := hub.Start(ctx, js, cred); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer hub.Stop()
|
||||
takeIns := &oneHandler{}
|
||||
if err := takeIns.Handle(pipeline.TakenInSubject, func(m jetstream.Msg) { heard <- string(m.Data()); _ = m.Ack() }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bindTakeIns(ctx, js, cred, takeIns.handle, sw, func(context.Context) bool { return false }, t.Logf)
|
||||
if sw.from.Load() == 0 {
|
||||
t.Fatal("the takeins durable is not bound")
|
||||
}
|
||||
// Said after the bind: the takeins durable's alone.
|
||||
if _, err := js.Publish(ctx, subject, []byte(`{"after":true}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
close(held)
|
||||
select {
|
||||
case d := <-worked:
|
||||
if d != `{"before":true}` {
|
||||
t.Errorf("the main durable worked %s, want the take-in said before the bind", d)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Fatal("the take-in said before the bind is worked by no durable")
|
||||
}
|
||||
select {
|
||||
case d := <-heard:
|
||||
if d != `{"after":true}` {
|
||||
t.Errorf("the takeins durable heard %s, want the take-in said after the bind", d)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Fatal("the take-in said after the bind is not heard on the takeins durable")
|
||||
}
|
||||
// The main durable acknowledged the second with no work: nothing more is worked once its queue is drained.
|
||||
select {
|
||||
case d := <-worked:
|
||||
t.Errorf("the main durable worked %s after the bind", d)
|
||||
case <-time.After(2 * time.Second):
|
||||
}
|
||||
info, err := events.Consumer(ctx, cred.EventConsumerName())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ci, err := info.Info(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ci.NumAckPending != 0 || ci.NumPending != 0 {
|
||||
t.Errorf("the main durable holds %d pending and %d unacknowledged, want none", ci.NumPending, ci.NumAckPending)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user