diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go index 103aac070..32c6ac8a2 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go @@ -31,7 +31,8 @@ import ( // A live publish enqueued behind 100 again items while the builder was away is sent first; a dropped item is never sent, and // put again within the duplicates window is not stored; the move of the old state run twice puts each item once, and the // builder removes the old state's bucket itself (H-A-461 (1)). The seat mesh-builds' verb publish-queue reads the stream: counts -// per holder and class, the oldest and newest, the live item listed first (H-A-454). Then §796's tag at another commit, its +// per holder and class, the oldest and newest, the live item listed first (H-A-454). An idle dispatch sends a live item put +// before an again one first, 50 rounds (H-A-473, third addendum). Then §796's tag at another commit, its // reservation's version-reserved waited on by name (§821), and // §794's backlog held six minutes, as steps (H-A-453 (3)). func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) { @@ -135,6 +136,55 @@ func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) { } mu.Unlock() + // An idle dispatch (H-A-473, its third addendum): its two pulls wait; a live item put, then an again one, is sent live first, + // every time, 50 rounds, each a holder of its own. + classCh := make(chan string, 2) + idle := &publishQueue{js: b.js, ackWait: time.Minute, logf: func(string, ...any) {}, binding: publishesOf("mesh-delivery-builder"), + take: func(_ context.Context, m itemMsg) { + var q versions.QueuedPublish + _ = json.Unmarshal(m.Data(), &q) + _ = m.Ack() + classCh <- q.Class + }} + for r := range 50 { + holder := fmt.Sprintf("idle%d", r) + idle.learn(ctx, holder) + for _, class := range []string{versions.PublishClassLive, versions.PublishClassStart} { // both pulls stand waiting + c, err := b.js.Consumer(ctx, bind.Stream, publishesConsumer(holder, class)) + if err != nil { + t.Fatal(err) + } + for { + info, err := c.Info(ctx) + if err != nil { + t.Fatal(err) + } + if info.NumWaiting > 0 { + break + } + } + } + for _, class := range []string{versions.PublishClassLive, versions.PublishClassStart} { + q := versions.QueuedPublish{V: 1, Holder: holder, Module: "m", Version: "0.1.0", Class: class, Queued: time.Now()} + data, _ := json.Marshal(q) + if _, err := b.js.Publish(ctx, idle.subject(bind, q), data); err != nil { + t.Fatal(err) + } + } + var order []string + for range 2 { + select { + case c := <-classCh: + order = append(order, c) + case <-time.After(30 * time.Second): + t.Fatalf("round %d: sent %v of a live and an again item", r, order) + } + } + if order[0] != versions.PublishClassLive { + t.Errorf("round %d: an idle dispatch sent %v, want the live item put first sent first", r, order) + } + } + // The move of the old state, run twice, puts each item once (at another holder, so the pull of oci is not asked). old, err := b.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: builds.StateBucket("mesh-delivery-builder", versions.PublishQueueState)}) if err != nil { diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go index 71cf6a75b..56b134604 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go @@ -637,7 +637,7 @@ func (pq *publishQueue) learn(ctx context.Context, holder string) { } // pull makes holder's two consumers and dispatches their items, the live class first, at most four sends at once: an again item -// waits while a live one waits on the queue. +// is run only when no live one waits on the queue or is in flight to the dispatch (dispatch, H-A-473). func (pq *publishQueue) pull(ctx context.Context, holder string) error { b, err := pq.binding() if err != nil { @@ -659,11 +659,12 @@ func (pq *publishQueue) pull(ctx context.Context, holder string) error { } // dispatch runs the items of one holder, at most four at once, every live item before an again one (design 47 H2 §793, -// H-A-473): it pulls the live consumer itself, a fetch of up to its free slots that does not wait, and pulls the again consumer -// only when that fetch returns nothing; an item is received the moment its fetch returns it, so no live item is delivered and -// not yet received while an again one is chosen. When neither holds an item, a pull of one item waits on each, and what it -// receives is run, the live first. A pull the bus refuses ends the dispatch, said; the holder is pulled again when an item of -// it is put. +// H-A-473 and its third addendum): it pulls the live consumer itself, a fetch of up to its free slots that does not wait, and +// pulls the again consumer only when that fetch returns nothing. When neither holds an item, a pull of one item waits on each, +// the bus's only wake for an item put or delivered again. An again item taken while the live pull waits is run only once the +// live consumer's delivered sequence is no further than the live items the dispatch received: a gap is a live item in flight +// to the waiting pull, and the again items are held until it arrives and runs. A pull the bus refuses ends the dispatch, said; +// the holder is pulled again when an item of it is put. func (pq *publishQueue) dispatch(ctx context.Context, holder string, live, again jetstream.Consumer) { sem := make(chan struct{}, versions.PublishSendsAtOnce) run := func(m jetstream.Msg) { @@ -681,6 +682,32 @@ func (pq *publishQueue) dispatch(ctx context.Context, holder string, live, again refused := func(err error) bool { return err != nil && !errors.Is(err, jetstream.ErrNoMessages) && !errors.Is(err, nats.ErrTimeout) && ctx.Err() == nil } + // received is the live consumer's highest delivery sequence the dispatch holds; it starts at what the consumer delivered + // before this dispatch, to pulls no longer there. + var received uint64 + if info, err := live.Info(ctx); err != nil { + if refused(err) { + failed(err) + return + } + } else { + received = info.Delivered.Consumer + } + gotLive := func(ms ...jetstream.Msg) { + for _, m := range ms { + if md, err := m.Metadata(); err == nil && md.Sequence.Consumer > received { + received = md.Sequence.Consumer + } + } + } + inFlight := func() bool { // a live item delivered to the waiting pull and not yet received + info, err := live.Info(ctx) + if err != nil { + pq.logf("the publishes of %s: the live consumer cannot be read (%v); the again items wait for the live pull's answer", holder, err) + return true + } + return info.Delivered.Consumer > received + } now := func(c jetstream.Consumer, n int) ([]jetstream.Msg, error) { b, err := c.FetchNoWait(n) if err != nil { @@ -711,19 +738,22 @@ func (pq *publishQueue) dispatch(ctx context.Context, holder string, live, again done <- w }() } - var heldLive, heldAgain jetstream.Msg // an item a waiting pull received + var heldLive jetstream.Msg // a live item the waiting pull received + var heldAgain []jetstream.Msg // again items received, not yet run var liveWaiting, againWaiting bool - heard := func(w waited, held *jetstream.Msg, waiting *bool) bool { + heard := func(w waited, waiting *bool, keep func(jetstream.Msg)) bool { *waiting = false if refused(w.err) { failed(w.err) return false } if w.m != nil { - *held = w.m + keep(w.m) } return true } + keepLive := func(m jetstream.Msg) { gotLive(m); heldLive = m } + keepAgain := func(m jetstream.Msg) { heldAgain = append(heldAgain, m) } for { select { case sem <- struct{}{}: @@ -742,41 +772,45 @@ func (pq *publishQueue) dispatch(ctx context.Context, holder string, live, again } select { case w := <-liveDone: - if !heard(w, &heldLive, &liveWaiting) { + if !heard(w, &liveWaiting, keepLive) { return } default: } select { case w := <-againDone: - if !heard(w, &heldAgain, &againWaiting) { + if !heard(w, &againWaiting, keepAgain) { return } default: } var got []jetstream.Msg - switch { - case heldLive != nil: + held := false // again items held for a live item in flight + if heldLive != nil { got, heldLive = []jetstream.Msg{heldLive}, nil - case !liveWaiting: + } else if !liveWaiting || len(heldAgain) > 0 { ms, err := now(live, n) if refused(err) { failed(err) return } + gotLive(ms...) got = ms } if len(got) == 0 { - switch { - case heldAgain != nil: - got, heldAgain = []jetstream.Msg{heldAgain}, nil - case !againWaiting: + if len(heldAgain) == 0 && !againWaiting { ms, err := now(again, n) if refused(err) { failed(err) return } - got = ms + heldAgain = ms + } + if len(heldAgain) > 0 && liveWaiting && inFlight() { + held = true + } else if len(heldAgain) > 0 { + k := min(n, len(heldAgain)) + got, heldAgain = heldAgain[:k], heldAgain[k:] } } for _, m := range got { @@ -792,17 +826,17 @@ func (pq *publishQueue) dispatch(ctx context.Context, holder string, live, again liveWaiting = true wait(live, liveDone) } - if !againWaiting { + if !againWaiting && !held { againWaiting = true wait(again, againDone) } select { case w := <-liveDone: - if !heard(w, &heldLive, &liveWaiting) { + if !heard(w, &liveWaiting, keepLive) { return } case w := <-againDone: - if !heard(w, &heldAgain, &againWaiting) { + if !heard(w, &againWaiting, keepAgain) { return } case <-ctx.Done():