Builder: an again item waits for a live one in flight to the idle dispatch's waiting pull

The idle dispatch keeps a waiting pull on each consumer, the bus's only wake for an
item put or delivered again. A live item the waiting pull was given but the dispatch
had not yet received let an again item go first (review finding on 1f33a4c0,
publishitem.go:700, 761, 797). An again item taken while the live pull waits now runs
only once the live consumer's delivered sequence is no further than the live items
received; a gap holds it until the live item arrives and runs.

Builds: design 47 H2 §793, H-A-473 (third addendum)
This commit is contained in:
2026-11-22 14:30:22 +01:00
parent 7f5f175b05
commit b77b70a66c
2 changed files with 107 additions and 23 deletions
@@ -31,7 +31,8 @@ import (
// A live publish enqueued behind 100 again items while the builder was away is sent first; a dropped item is never sent, and
// put again within the duplicates window is not stored; the move of the old state run twice puts each item once, and the
// builder removes the old state's bucket itself (H-A-461 (1)). The seat mesh-builds' verb publish-queue reads the stream: counts
// per holder and class, the oldest and newest, the live item listed first (H-A-454). Then §796's tag at another commit, its
// per holder and class, the oldest and newest, the live item listed first (H-A-454). An idle dispatch sends a live item put
// before an again one first, 50 rounds (H-A-473, third addendum). Then §796's tag at another commit, its
// reservation's version-reserved waited on by name (§821), and
// §794's backlog held six minutes, as steps (H-A-453 (3)).
func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) {
@@ -135,6 +136,55 @@ func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) {
}
mu.Unlock()
// An idle dispatch (H-A-473, its third addendum): its two pulls wait; a live item put, then an again one, is sent live first,
// every time, 50 rounds, each a holder of its own.
classCh := make(chan string, 2)
idle := &publishQueue{js: b.js, ackWait: time.Minute, logf: func(string, ...any) {}, binding: publishesOf("mesh-delivery-builder"),
take: func(_ context.Context, m itemMsg) {
var q versions.QueuedPublish
_ = json.Unmarshal(m.Data(), &q)
_ = m.Ack()
classCh <- q.Class
}}
for r := range 50 {
holder := fmt.Sprintf("idle%d", r)
idle.learn(ctx, holder)
for _, class := range []string{versions.PublishClassLive, versions.PublishClassStart} { // both pulls stand waiting
c, err := b.js.Consumer(ctx, bind.Stream, publishesConsumer(holder, class))
if err != nil {
t.Fatal(err)
}
for {
info, err := c.Info(ctx)
if err != nil {
t.Fatal(err)
}
if info.NumWaiting > 0 {
break
}
}
}
for _, class := range []string{versions.PublishClassLive, versions.PublishClassStart} {
q := versions.QueuedPublish{V: 1, Holder: holder, Module: "m", Version: "0.1.0", Class: class, Queued: time.Now()}
data, _ := json.Marshal(q)
if _, err := b.js.Publish(ctx, idle.subject(bind, q), data); err != nil {
t.Fatal(err)
}
}
var order []string
for range 2 {
select {
case c := <-classCh:
order = append(order, c)
case <-time.After(30 * time.Second):
t.Fatalf("round %d: sent %v of a live and an again item", r, order)
}
}
if order[0] != versions.PublishClassLive {
t.Errorf("round %d: an idle dispatch sent %v, want the live item put first sent first", r, order)
}
}
// The move of the old state, run twice, puts each item once (at another holder, so the pull of oci is not asked).
old, err := b.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: builds.StateBucket("mesh-delivery-builder", versions.PublishQueueState)})
if err != nil {
@@ -637,7 +637,7 @@ func (pq *publishQueue) learn(ctx context.Context, holder string) {
}
// pull makes holder's two consumers and dispatches their items, the live class first, at most four sends at once: an again item
// waits while a live one waits on the queue.
// is run only when no live one waits on the queue or is in flight to the dispatch (dispatch, H-A-473).
func (pq *publishQueue) pull(ctx context.Context, holder string) error {
b, err := pq.binding()
if err != nil {
@@ -659,11 +659,12 @@ func (pq *publishQueue) pull(ctx context.Context, holder string) error {
}
// dispatch runs the items of one holder, at most four at once, every live item before an again one (design 47 H2 §793,
// H-A-473): it pulls the live consumer itself, a fetch of up to its free slots that does not wait, and pulls the again consumer
// only when that fetch returns nothing; an item is received the moment its fetch returns it, so no live item is delivered and
// not yet received while an again one is chosen. When neither holds an item, a pull of one item waits on each, and what it
// receives is run, the live first. A pull the bus refuses ends the dispatch, said; the holder is pulled again when an item of
// it is put.
// H-A-473 and its third addendum): it pulls the live consumer itself, a fetch of up to its free slots that does not wait, and
// pulls the again consumer only when that fetch returns nothing. When neither holds an item, a pull of one item waits on each,
// the bus's only wake for an item put or delivered again. An again item taken while the live pull waits is run only once the
// live consumer's delivered sequence is no further than the live items the dispatch received: a gap is a live item in flight
// to the waiting pull, and the again items are held until it arrives and runs. A pull the bus refuses ends the dispatch, said;
// the holder is pulled again when an item of it is put.
func (pq *publishQueue) dispatch(ctx context.Context, holder string, live, again jetstream.Consumer) {
sem := make(chan struct{}, versions.PublishSendsAtOnce)
run := func(m jetstream.Msg) {
@@ -681,6 +682,32 @@ func (pq *publishQueue) dispatch(ctx context.Context, holder string, live, again
refused := func(err error) bool {
return err != nil && !errors.Is(err, jetstream.ErrNoMessages) && !errors.Is(err, nats.ErrTimeout) && ctx.Err() == nil
}
// received is the live consumer's highest delivery sequence the dispatch holds; it starts at what the consumer delivered
// before this dispatch, to pulls no longer there.
var received uint64
if info, err := live.Info(ctx); err != nil {
if refused(err) {
failed(err)
return
}
} else {
received = info.Delivered.Consumer
}
gotLive := func(ms ...jetstream.Msg) {
for _, m := range ms {
if md, err := m.Metadata(); err == nil && md.Sequence.Consumer > received {
received = md.Sequence.Consumer
}
}
}
inFlight := func() bool { // a live item delivered to the waiting pull and not yet received
info, err := live.Info(ctx)
if err != nil {
pq.logf("the publishes of %s: the live consumer cannot be read (%v); the again items wait for the live pull's answer", holder, err)
return true
}
return info.Delivered.Consumer > received
}
now := func(c jetstream.Consumer, n int) ([]jetstream.Msg, error) {
b, err := c.FetchNoWait(n)
if err != nil {
@@ -711,19 +738,22 @@ func (pq *publishQueue) dispatch(ctx context.Context, holder string, live, again
done <- w
}()
}
var heldLive, heldAgain jetstream.Msg // an item a waiting pull received
var heldLive jetstream.Msg // a live item the waiting pull received
var heldAgain []jetstream.Msg // again items received, not yet run
var liveWaiting, againWaiting bool
heard := func(w waited, held *jetstream.Msg, waiting *bool) bool {
heard := func(w waited, waiting *bool, keep func(jetstream.Msg)) bool {
*waiting = false
if refused(w.err) {
failed(w.err)
return false
}
if w.m != nil {
*held = w.m
keep(w.m)
}
return true
}
keepLive := func(m jetstream.Msg) { gotLive(m); heldLive = m }
keepAgain := func(m jetstream.Msg) { heldAgain = append(heldAgain, m) }
for {
select {
case sem <- struct{}{}:
@@ -742,41 +772,45 @@ func (pq *publishQueue) dispatch(ctx context.Context, holder string, live, again
}
select {
case w := <-liveDone:
if !heard(w, &heldLive, &liveWaiting) {
if !heard(w, &liveWaiting, keepLive) {
return
}
default:
}
select {
case w := <-againDone:
if !heard(w, &heldAgain, &againWaiting) {
if !heard(w, &againWaiting, keepAgain) {
return
}
default:
}
var got []jetstream.Msg
switch {
case heldLive != nil:
held := false // again items held for a live item in flight
if heldLive != nil {
got, heldLive = []jetstream.Msg{heldLive}, nil
case !liveWaiting:
} else if !liveWaiting || len(heldAgain) > 0 {
ms, err := now(live, n)
if refused(err) {
failed(err)
return
}
gotLive(ms...)
got = ms
}
if len(got) == 0 {
switch {
case heldAgain != nil:
got, heldAgain = []jetstream.Msg{heldAgain}, nil
case !againWaiting:
if len(heldAgain) == 0 && !againWaiting {
ms, err := now(again, n)
if refused(err) {
failed(err)
return
}
got = ms
heldAgain = ms
}
if len(heldAgain) > 0 && liveWaiting && inFlight() {
held = true
} else if len(heldAgain) > 0 {
k := min(n, len(heldAgain))
got, heldAgain = heldAgain[:k], heldAgain[k:]
}
}
for _, m := range got {
@@ -792,17 +826,17 @@ func (pq *publishQueue) dispatch(ctx context.Context, holder string, live, again
liveWaiting = true
wait(live, liveDone)
}
if !againWaiting {
if !againWaiting && !held {
againWaiting = true
wait(again, againDone)
}
select {
case w := <-liveDone:
if !heard(w, &heldLive, &liveWaiting) {
if !heard(w, &liveWaiting, keepLive) {
return
}
case w := <-againDone:
if !heard(w, &heldAgain, &againWaiting) {
if !heard(w, &againWaiting, keepAgain) {
return
}
case <-ctx.Done():