Builder: an again item waits for a live one in flight to the idle dispatch's waiting pull
The idle dispatch keeps a waiting pull on each consumer, the bus's only wake for an
item put or delivered again. A live item the waiting pull was given but the dispatch
had not yet received let an again item go first (review finding on 1f33a4c0,
publishitem.go:700, 761, 797). An again item taken while the live pull waits now runs
only once the live consumer's delivered sequence is no further than the live items
received; a gap holds it until the live item arrives and runs.
Builds: design 47 H2 §793, H-A-473 (third addendum)
This commit is contained in:
+51
-1
@@ -31,7 +31,8 @@ import (
|
||||
// A live publish enqueued behind 100 again items while the builder was away is sent first; a dropped item is never sent, and
|
||||
// put again within the duplicates window is not stored; the move of the old state run twice puts each item once, and the
|
||||
// builder removes the old state's bucket itself (H-A-461 (1)). The seat mesh-builds' verb publish-queue reads the stream: counts
|
||||
// per holder and class, the oldest and newest, the live item listed first (H-A-454). Then §796's tag at another commit, its
|
||||
// per holder and class, the oldest and newest, the live item listed first (H-A-454). An idle dispatch sends a live item put
|
||||
// before an again one first, 50 rounds (H-A-473, third addendum). Then §796's tag at another commit, its
|
||||
// reservation's version-reserved waited on by name (§821), and
|
||||
// §794's backlog held six minutes, as steps (H-A-453 (3)).
|
||||
func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) {
|
||||
@@ -135,6 +136,55 @@ func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) {
|
||||
}
|
||||
mu.Unlock()
|
||||
|
||||
// An idle dispatch (H-A-473, its third addendum): its two pulls wait; a live item put, then an again one, is sent live first,
|
||||
// every time, 50 rounds, each a holder of its own.
|
||||
classCh := make(chan string, 2)
|
||||
idle := &publishQueue{js: b.js, ackWait: time.Minute, logf: func(string, ...any) {}, binding: publishesOf("mesh-delivery-builder"),
|
||||
take: func(_ context.Context, m itemMsg) {
|
||||
var q versions.QueuedPublish
|
||||
_ = json.Unmarshal(m.Data(), &q)
|
||||
_ = m.Ack()
|
||||
classCh <- q.Class
|
||||
}}
|
||||
for r := range 50 {
|
||||
holder := fmt.Sprintf("idle%d", r)
|
||||
idle.learn(ctx, holder)
|
||||
for _, class := range []string{versions.PublishClassLive, versions.PublishClassStart} { // both pulls stand waiting
|
||||
c, err := b.js.Consumer(ctx, bind.Stream, publishesConsumer(holder, class))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for {
|
||||
info, err := c.Info(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.NumWaiting > 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, class := range []string{versions.PublishClassLive, versions.PublishClassStart} {
|
||||
q := versions.QueuedPublish{V: 1, Holder: holder, Module: "m", Version: "0.1.0", Class: class, Queued: time.Now()}
|
||||
data, _ := json.Marshal(q)
|
||||
if _, err := b.js.Publish(ctx, idle.subject(bind, q), data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var order []string
|
||||
for range 2 {
|
||||
select {
|
||||
case c := <-classCh:
|
||||
order = append(order, c)
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatalf("round %d: sent %v of a live and an again item", r, order)
|
||||
}
|
||||
}
|
||||
if order[0] != versions.PublishClassLive {
|
||||
t.Errorf("round %d: an idle dispatch sent %v, want the live item put first sent first", r, order)
|
||||
}
|
||||
}
|
||||
|
||||
// The move of the old state, run twice, puts each item once (at another holder, so the pull of oci is not asked).
|
||||
old, err := b.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: builds.StateBucket("mesh-delivery-builder", versions.PublishQueueState)})
|
||||
if err != nil {
|
||||
|
||||
@@ -637,7 +637,7 @@ func (pq *publishQueue) learn(ctx context.Context, holder string) {
|
||||
}
|
||||
|
||||
// pull makes holder's two consumers and dispatches their items, the live class first, at most four sends at once: an again item
|
||||
// waits while a live one waits on the queue.
|
||||
// is run only when no live one waits on the queue or is in flight to the dispatch (dispatch, H-A-473).
|
||||
func (pq *publishQueue) pull(ctx context.Context, holder string) error {
|
||||
b, err := pq.binding()
|
||||
if err != nil {
|
||||
@@ -659,11 +659,12 @@ func (pq *publishQueue) pull(ctx context.Context, holder string) error {
|
||||
}
|
||||
|
||||
// dispatch runs the items of one holder, at most four at once, every live item before an again one (design 47 H2 §793,
|
||||
// H-A-473): it pulls the live consumer itself, a fetch of up to its free slots that does not wait, and pulls the again consumer
|
||||
// only when that fetch returns nothing; an item is received the moment its fetch returns it, so no live item is delivered and
|
||||
// not yet received while an again one is chosen. When neither holds an item, a pull of one item waits on each, and what it
|
||||
// receives is run, the live first. A pull the bus refuses ends the dispatch, said; the holder is pulled again when an item of
|
||||
// it is put.
|
||||
// H-A-473 and its third addendum): it pulls the live consumer itself, a fetch of up to its free slots that does not wait, and
|
||||
// pulls the again consumer only when that fetch returns nothing. When neither holds an item, a pull of one item waits on each,
|
||||
// the bus's only wake for an item put or delivered again. An again item taken while the live pull waits is run only once the
|
||||
// live consumer's delivered sequence is no further than the live items the dispatch received: a gap is a live item in flight
|
||||
// to the waiting pull, and the again items are held until it arrives and runs. A pull the bus refuses ends the dispatch, said;
|
||||
// the holder is pulled again when an item of it is put.
|
||||
func (pq *publishQueue) dispatch(ctx context.Context, holder string, live, again jetstream.Consumer) {
|
||||
sem := make(chan struct{}, versions.PublishSendsAtOnce)
|
||||
run := func(m jetstream.Msg) {
|
||||
@@ -681,6 +682,32 @@ func (pq *publishQueue) dispatch(ctx context.Context, holder string, live, again
|
||||
refused := func(err error) bool {
|
||||
return err != nil && !errors.Is(err, jetstream.ErrNoMessages) && !errors.Is(err, nats.ErrTimeout) && ctx.Err() == nil
|
||||
}
|
||||
// received is the live consumer's highest delivery sequence the dispatch holds; it starts at what the consumer delivered
|
||||
// before this dispatch, to pulls no longer there.
|
||||
var received uint64
|
||||
if info, err := live.Info(ctx); err != nil {
|
||||
if refused(err) {
|
||||
failed(err)
|
||||
return
|
||||
}
|
||||
} else {
|
||||
received = info.Delivered.Consumer
|
||||
}
|
||||
gotLive := func(ms ...jetstream.Msg) {
|
||||
for _, m := range ms {
|
||||
if md, err := m.Metadata(); err == nil && md.Sequence.Consumer > received {
|
||||
received = md.Sequence.Consumer
|
||||
}
|
||||
}
|
||||
}
|
||||
inFlight := func() bool { // a live item delivered to the waiting pull and not yet received
|
||||
info, err := live.Info(ctx)
|
||||
if err != nil {
|
||||
pq.logf("the publishes of %s: the live consumer cannot be read (%v); the again items wait for the live pull's answer", holder, err)
|
||||
return true
|
||||
}
|
||||
return info.Delivered.Consumer > received
|
||||
}
|
||||
now := func(c jetstream.Consumer, n int) ([]jetstream.Msg, error) {
|
||||
b, err := c.FetchNoWait(n)
|
||||
if err != nil {
|
||||
@@ -711,19 +738,22 @@ func (pq *publishQueue) dispatch(ctx context.Context, holder string, live, again
|
||||
done <- w
|
||||
}()
|
||||
}
|
||||
var heldLive, heldAgain jetstream.Msg // an item a waiting pull received
|
||||
var heldLive jetstream.Msg // a live item the waiting pull received
|
||||
var heldAgain []jetstream.Msg // again items received, not yet run
|
||||
var liveWaiting, againWaiting bool
|
||||
heard := func(w waited, held *jetstream.Msg, waiting *bool) bool {
|
||||
heard := func(w waited, waiting *bool, keep func(jetstream.Msg)) bool {
|
||||
*waiting = false
|
||||
if refused(w.err) {
|
||||
failed(w.err)
|
||||
return false
|
||||
}
|
||||
if w.m != nil {
|
||||
*held = w.m
|
||||
keep(w.m)
|
||||
}
|
||||
return true
|
||||
}
|
||||
keepLive := func(m jetstream.Msg) { gotLive(m); heldLive = m }
|
||||
keepAgain := func(m jetstream.Msg) { heldAgain = append(heldAgain, m) }
|
||||
for {
|
||||
select {
|
||||
case sem <- struct{}{}:
|
||||
@@ -742,41 +772,45 @@ func (pq *publishQueue) dispatch(ctx context.Context, holder string, live, again
|
||||
}
|
||||
select {
|
||||
case w := <-liveDone:
|
||||
if !heard(w, &heldLive, &liveWaiting) {
|
||||
if !heard(w, &liveWaiting, keepLive) {
|
||||
return
|
||||
}
|
||||
default:
|
||||
}
|
||||
select {
|
||||
case w := <-againDone:
|
||||
if !heard(w, &heldAgain, &againWaiting) {
|
||||
if !heard(w, &againWaiting, keepAgain) {
|
||||
return
|
||||
}
|
||||
default:
|
||||
}
|
||||
var got []jetstream.Msg
|
||||
switch {
|
||||
case heldLive != nil:
|
||||
held := false // again items held for a live item in flight
|
||||
if heldLive != nil {
|
||||
got, heldLive = []jetstream.Msg{heldLive}, nil
|
||||
case !liveWaiting:
|
||||
} else if !liveWaiting || len(heldAgain) > 0 {
|
||||
ms, err := now(live, n)
|
||||
if refused(err) {
|
||||
failed(err)
|
||||
return
|
||||
}
|
||||
gotLive(ms...)
|
||||
got = ms
|
||||
}
|
||||
if len(got) == 0 {
|
||||
switch {
|
||||
case heldAgain != nil:
|
||||
got, heldAgain = []jetstream.Msg{heldAgain}, nil
|
||||
case !againWaiting:
|
||||
if len(heldAgain) == 0 && !againWaiting {
|
||||
ms, err := now(again, n)
|
||||
if refused(err) {
|
||||
failed(err)
|
||||
return
|
||||
}
|
||||
got = ms
|
||||
heldAgain = ms
|
||||
}
|
||||
if len(heldAgain) > 0 && liveWaiting && inFlight() {
|
||||
held = true
|
||||
} else if len(heldAgain) > 0 {
|
||||
k := min(n, len(heldAgain))
|
||||
got, heldAgain = heldAgain[:k], heldAgain[k:]
|
||||
}
|
||||
}
|
||||
for _, m := range got {
|
||||
@@ -792,17 +826,17 @@ func (pq *publishQueue) dispatch(ctx context.Context, holder string, live, again
|
||||
liveWaiting = true
|
||||
wait(live, liveDone)
|
||||
}
|
||||
if !againWaiting {
|
||||
if !againWaiting && !held {
|
||||
againWaiting = true
|
||||
wait(again, againDone)
|
||||
}
|
||||
select {
|
||||
case w := <-liveDone:
|
||||
if !heard(w, &heldLive, &liveWaiting) {
|
||||
if !heard(w, &liveWaiting, keepLive) {
|
||||
return
|
||||
}
|
||||
case w := <-againDone:
|
||||
if !heard(w, &heldAgain, &againWaiting) {
|
||||
if !heard(w, &againWaiting, keepAgain) {
|
||||
return
|
||||
}
|
||||
case <-ctx.Done():
|
||||
|
||||
Reference in New Issue
Block a user