diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go index f10fd3886..f69aa98d7 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go @@ -28,15 +28,15 @@ import ( "mesh-delivery-builder/internal/versions" ) -// Design 47 H2 §793 (whole, against the bus): the builder's work queue publishes, its pull consumers made as it learns a holder. -// A live publish enqueued behind 100 again items while the builder was away is sent first; a dropped item is never sent, and -// put again within the duplicates window is not stored; the move of the old state run twice puts each item once, and the -// builder removes the old state's bucket itself (H-A-461 (1)). The seat mesh-builds' verb publish-queue reads the stream: counts -// per holder and class, the oldest and newest, the live item listed first (H-A-454). An idle dispatch sends a live item put -// before an again one first, 50 rounds (H-A-473, third addendum). Then §796's tag at another commit, its -// reservation's version-reserved waited on by name (§821), and -// §794's backlog held six minutes, as steps (H-A-453 (3)). -func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) { +// Design 47 H2 §793 (whole, against the bus): the builder's work queue publishes, one pull consumer made as it learns a holder +// (H-A-483 and its addenda), the class durables of an earlier release retired. No item is lost, a failed send comes back after +// its pause behind an item put meanwhile, and a dropped item never comes back; 100 again items and a live one put while the +// builder was away are all sent, the dropped one never, and put again within the duplicates window it is not stored; the move +// of the old state run twice puts each item once, and the builder removes the old state's bucket itself (H-A-461 (1)). The seat +// mesh-builds' verb publish-queue reads the stream: counts per holder and class, the oldest and newest, the live item listed +// first (H-A-454). Then §796's tag at another commit, its reservation's version-reserved waited on by name (§821), and §794's +// backlog held six minutes, as steps (H-A-453 (3)). +func TestThePublishQueueLosesNoItemAndNeverSendsADroppedItem(t *testing.T) { b := startBed(t) ctx := b.ctx // The stream as the manifest's old streams section declares it: the module's own, retention work-queue, duplicates 24 hours. @@ -45,66 +45,153 @@ func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) { Retention: jetstream.WorkQueuePolicy, Discard: jetstream.DiscardNew, Duplicates: 24 * time.Hour, MaxAge: 7 * 24 * time.Hour}); err != nil { t.Fatal(err) } - // An idle dispatch (H-A-473, its third addendum): its two pulls wait; a live item put, then an again one, is sent live first; - // and an again item put after a live one, while the dispatch runs an again item put before both, is sent after the live one. - // Every time, 50 rounds, each a holder of its own. - sentOf := make(chan string, 8) - idle := &publishQueue{js: b.js, ackWait: time.Minute, logf: func(string, ...any) {}, binding: publishesOf("mesh-delivery-builder"), - take: func(_ context.Context, m itemMsg) { - var q versions.QueuedPublish - _ = json.Unmarshal(m.Data(), &q) - _ = m.Ack() - sentOf <- q.Module - }} - t.Run("idle dispatch", func(t *testing.T) { - roundBound := testbus.SetupBound(t) // the level's set-up share (MESH_TEST_LEVEL_BOUND), as the other whole tests - for r := range 50 { - holder := fmt.Sprintf("idle%d", r) - idle.learn(ctx, holder) - for _, class := range []string{versions.PublishClassLive, versions.PublishClassStart} { // both pulls stand waiting - c, err := b.js.Consumer(ctx, bind.Stream, publishesConsumer(holder, class)) - if err != nil { - t.Fatal(err) + // One consumer per holder (H-A-483 and its addenda): the holder's class durables of an earlier release are retired, the item + // one of them held unacknowledged reaches the one consumer; no item is lost; a failed send comes back after its pause, and a + // live item put while it waits is sent before it returns; a dropped item never comes back. + t.Run("one consumer", func(t *testing.T) { + const holder, pause = "paused", 3 * time.Second // the pause is far longer than a send of an item put now + bound := testbus.SetupBound(t) + kv := &itemKV{vals: map[string][]byte{}, revs: map[string]uint64{}} + type send struct { + module string + at time.Time + } + var mu sync.Mutex + published, failed := map[string]bool{}, map[string]bool{} + sentCh := make(chan send, 16) + items := &publishItems{reasks: func() jetstream.KeyValue { return kv }, set: publishSettings{Pause: pause, PauseMax: time.Minute, Tries: 5}, + now: time.Now, logf: t.Logf, + reserved: func(_ context.Context, m, v string) (versions.ReservedVersion, bool, error) { + mu.Lock() + defer mu.Unlock() + return versions.ReservedVersion{Module: m, Version: v, Artifacts: map[string]string{m + ":a": "sha256:a"}, Published: map[string]bool{m + ":a": published[m]}, BuiltAt: time.Now()}, true, nil + }, + send: func(_ context.Context, q versions.QueuedPublish) versions.SendFate { + sentCh <- send{q.Module, time.Now()} + mu.Lock() + defer mu.Unlock() + if (q.Module == "fail" || q.Module == "dropme") && !failed[q.Module] { // the first send of each fails + failed[q.Module] = true + return versions.SendFate{Failed: "the registry is down"} } - for { - info, err := c.Info(ctx) - if err != nil { - t.Fatal(err) - } - if info.NumWaiting > 0 { - break - } + published[q.Module] = true + return versions.SendFate{Sent: true} + }} + pq := &publishQueue{js: b.js, ackWait: time.Minute, logf: t.Logf, take: items.take, binding: publishesOf("mesh-delivery-builder")} + put := func(module, class string) { + q := versions.QueuedPublish{V: 1, Holder: holder, Module: module, Version: "0.1.0", Artifacts: map[string]string{module + ":a": "sha256:a"}, Class: class, Queued: time.Now()} + data, _ := json.Marshal(q) + if _, err := b.js.Publish(ctx, pq.subject(bind, q), data); err != nil { + t.Fatal(err) + } + } + hear := func() { // each publish sent is heard: its item acknowledged once it is held + mu.Lock() + var done []string + for m, ok := range published { + if ok { + done = append(done, m) } } - put := func(module, class string) { - q := versions.QueuedPublish{V: 1, Holder: holder, Module: module, Version: "0.1.0", Class: class, Queued: time.Now()} - data, _ := json.Marshal(q) - if _, err := b.js.Publish(ctx, idle.subject(bind, q), data); err != nil { - t.Fatal(err) - } + mu.Unlock() + for _, m := range done { + items.heard(ctx, m, "0.1.0") } - sent := func(n int) []string { - var order []string - for range n { - select { - case m := <-sentOf: - order = append(order, m) - case <-time.After(roundBound): - t.Fatalf("round %d: sent %v of %d items", r, order, n) - } - } - return order + } + next := func() send { + t.Helper() + select { + case s := <-sentCh: + hear() + return s + case <-time.After(bound): + t.Fatal("no item was sent within the level's set-up share") } - put("live", versions.PublishClassLive) - put("again", versions.PublishClassStart) - if order := sent(2); order[0] != "live" { - t.Errorf("round %d: an idle dispatch sent %v, want the live item put first sent first", r, order) + return send{} + } + // The two class durables an earlier release made, the live one holding an item given and not acknowledged. + for _, class := range []string{versions.PublishClassLive, versions.PublishClassStart} { + if _, err := b.js.CreateOrUpdateConsumer(ctx, bind.Stream, jetstream.ConsumerConfig{Durable: publishesConsumer(holder) + "-" + class, + FilterSubject: bind.Subject + ".publish." + holder + "." + class + ".>", AckPolicy: jetstream.AckExplicitPolicy, AckWait: time.Hour}); err != nil { + t.Fatal(err) } - put("before", versions.PublishClassStart) - put("live2", versions.PublishClassLive) - put("after", versions.PublishClassStart) - if order := sent(3); slices.Index(order, "after") < slices.Index(order, "live2") { - t.Errorf("round %d: the dispatch sent %v, want the again item put after the live one sent after it", r, order) + } + put("given", versions.PublishClassLive) + old, err := b.js.Consumer(ctx, bind.Stream, publishesConsumer(holder)+"-"+versions.PublishClassLive) + if err != nil { + t.Fatal(err) + } + if m, err := old.Next(); err != nil { + t.Fatal(err) + } else if md, _ := m.Metadata(); md.NumDelivered != 1 { + t.Fatalf("the old live durable gave %+v", md) + } + pq.learn(ctx, holder) + stream, err := b.js.Stream(ctx, bind.Stream) + if err != nil { + t.Fatal(err) + } + var names []string + for n := range stream.ConsumerNames(ctx).Name() { + if strings.HasPrefix(n, publishesConsumer(holder)) { + names = append(names, n) + } + } + if !slices.Equal(names, []string{publishesConsumer(holder)}) { + t.Fatalf("the consumers of %s are %v, want only %s: the class durables retired", holder, names, publishesConsumer(holder)) + } + if s := next(); s.module != "given" { + t.Fatalf("sent %s first, want the item the old durable held unacknowledged", s.module) + } + put("fail", versions.PublishClassStart) + put("dropme", versions.PublishClassStart) + var firstFail time.Time + for range 2 { + if s := next(); s.module == "fail" { + firstFail = s.at + } + } + put("live", versions.PublishClassLive) // put while both failed items wait out their pause + if s := next(); s.module != "live" { + t.Fatalf("sent %s, want the live item put while the failed ones wait their pause", s.module) + } + if answer, err := pq.drop(ctx, items, "dropme", "0.1.0", "op@node", "a test drops it"); err != nil { + t.Fatalf("drop: %v", err) + } else { + t.Log(answer) + } + if s := next(); s.module != "fail" || s.at.Sub(firstFail) < pause { + t.Fatalf("sent %s %s after the failed send, want fail again after its pause %s", s.module, s.at.Sub(firstFail), pause) + } + // Every item is acknowledged and the dropped one is gone: nothing is left to come back. + c, err := b.js.Consumer(ctx, bind.Stream, publishesConsumer(holder)) + if err != nil { + t.Fatal(err) + } + st, err := b.js.Stream(ctx, bind.Stream) + if err != nil { + t.Fatal(err) + } + for deadline := time.Now().Add(bound); ; { + hear() + info, err := c.Info(ctx) + if err != nil { + t.Fatal(err) + } + sinfo, err := st.Info(ctx, jetstream.WithSubjectFilter(bind.Subject+".publish."+holder+".>")) + if err != nil { + t.Fatal(err) + } + if info.NumAckPending == 0 && info.NumPending == 0 && len(sinfo.State.Subjects) == 0 { + break + } + if time.Now().After(deadline) { + t.Fatalf("the queue of %s holds %v, its consumer %d pending and %d unacknowledged: want every item acknowledged and the dropped one gone", holder, sinfo.State.Subjects, info.NumPending, info.NumAckPending) + } + select { + case s := <-sentCh: + t.Fatalf("sent %s again, want nothing more", s.module) + case <-time.After(10 * time.Millisecond): } } }) @@ -161,15 +248,7 @@ func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) { if err := pq.learnAll(ctx); err != nil { t.Fatal(err) } - select { - case first := <-sentCh: - if first != "live" { - t.Errorf("the first item sent is %s, want the live one, enqueued behind 100 again items", first) - } - case <-time.After(30 * time.Second): - t.Fatal("nothing was sent") - } - for range 4 { // the again items in flight: at most four held unacknowledged, their publish unheard + for range 4 { // the items in flight: at most four held unacknowledged, their publish unheard select { case <-sentCh: case <-time.After(30 * time.Second): diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go index d2ad1ff18..1463e2d47 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go @@ -531,9 +531,9 @@ func builtOf(all []versions.ReservedVersion) []versions.ReservedVersion { // the manifest's old streams section with the work-queue retention, the module's own, consumed by the module's one process on the // module's account (H-A-464), which declares no bus and so is the stream's one reader by the module's shape; each item on // `mesh.stream..event.publish....` under the message id -// `...`. The builder makes the pull consumers at run time as it learns a holder, -// `publishes--`, with ack wait publish-pause-max, max ack pending four and max deliveries ten, and pulls the live -// class first, at most four sends at once per holder. +// `...`. The builder makes one pull consumer at run time as it learns a holder, +// `publishes-`, with ack wait publish-pause-max, max ack pending four and max deliveries ten, and sends its items in the +// order the consumer gives them, at most four at once per holder (H-A-483 and its addenda). type publishQueue struct { js jetstream.JetStream binding func() (publishesStream, error) @@ -555,16 +555,16 @@ func publishesOf(module string) func() (publishesStream, error) { return func() (publishesStream, error) { return b, nil } } -// The consumers' numbers (H-A-446 (2)): at most four sends in flight per holder and class, and ten deliveries, the work queue's -// largest; the builder ends an item at publish-tries anyway. +// The consumer's numbers (H-A-446 (2)): at most four sends in flight per holder, and ten deliveries, the work queue's largest; +// the builder ends an item at publish-tries anyway. const ( publishesMaxAckPending = versions.PublishSendsAtOnce publishesMaxDeliver = 10 ) -// publishesConsumer is the durable consumer of holder's items of class. -func publishesConsumer(holder, class string) string { - return versions.PublishesLocal + "-" + holder + "-" + class +// publishesConsumer is the one durable consumer of holder's items (H-A-483 addendum). +func publishesConsumer(holder string) string { + return versions.PublishesLocal + "-" + holder } // subject is the item's subject on the bus. @@ -636,223 +636,71 @@ func (pq *publishQueue) learn(ctx context.Context, holder string) { } } -// pull makes holder's two consumers and dispatches their items, the live class first, at most four sends at once: an again item -// is run only when no live one waits on the queue or is in flight to the dispatch (dispatch, H-A-473). +// pull makes holder's one consumer and dispatches its items (design 47 H2 §793, H-A-483 and its addenda): first it retires the +// holder's class durables an earlier release made (`-live`, `-again`), whose filters a work queue refuses beside the one; their +// unacknowledged items stay on the queue and reach the one consumer. func (pq *publishQueue) pull(ctx context.Context, holder string) error { b, err := pq.binding() if err != nil { return err } - cons := map[string]jetstream.Consumer{} - for _, class := range []string{versions.PublishClassStart, versions.PublishClassLive} { - c, err := pq.js.CreateOrUpdateConsumer(ctx, b.Stream, jetstream.ConsumerConfig{Durable: publishesConsumer(holder, class), - FilterSubject: b.Subject + ".publish." + versions.EscapeToken(holder) + "." + class + ".>", AckPolicy: jetstream.AckExplicitPolicy, - AckWait: pq.ackWait, MaxAckPending: publishesMaxAckPending, MaxDeliver: publishesMaxDeliver}) - if err != nil { - return fmt.Errorf("the consumer %s: %w", publishesConsumer(holder, class), err) + for _, class := range []string{versions.PublishClassLive, versions.PublishClassStart} { + old := publishesConsumer(holder) + "-" + class + err := pq.js.DeleteConsumer(ctx, b.Stream, old) + switch { + case err == nil: + pq.logf("the consumer %s of an earlier release is removed: the publishes of %s have one consumer (H-A-483)", old, holder) + case !errors.Is(err, jetstream.ErrConsumerNotFound): + return fmt.Errorf("the consumer %s of an earlier release is not removed: %w", old, err) } - cons[class] = c } - pq.logf("pulling the publishes of %s, the live first", holder) - go pq.dispatch(ctx, holder, cons[versions.PublishClassLive], cons[versions.PublishClassStart]) + c, err := pq.js.CreateOrUpdateConsumer(ctx, b.Stream, jetstream.ConsumerConfig{Durable: publishesConsumer(holder), + FilterSubject: b.Subject + ".publish." + versions.EscapeToken(holder) + ".>", AckPolicy: jetstream.AckExplicitPolicy, + AckWait: pq.ackWait, MaxAckPending: publishesMaxAckPending, MaxDeliver: publishesMaxDeliver}) + if err != nil { + return fmt.Errorf("the consumer %s: %w", publishesConsumer(holder), err) + } + pq.logf("pulling the publishes of %s", holder) + go pq.dispatch(ctx, holder, c) return nil } -// dispatch runs the items of one holder, at most four at once, every live item before an again one (design 47 H2 §793, -// H-A-473 and its third addendum): on every turn no live item is held it fetches live without waiting, up to its free slots, -// and takes again items only when that fetch returned nothing. With again items held it fetches live once more; only when -// that returns nothing are they run, and while the live pull waits only once the live consumer's delivered sequence is no -// further than the live items received: a gap is a live item in flight to the waiting pull, and the again items wait until it -// arrives and runs. When neither holds an item, a pull of one item waits on each, the bus's only wake for an item put or -// delivered again. A pull the bus refuses ends the dispatch, said; the holder is pulled again when an item of it is put. -func (pq *publishQueue) dispatch(ctx context.Context, holder string, live, again jetstream.Consumer) { +// dispatch runs the items of one holder in the order its one consumer gives them, at most four at once (design 47 H2 §793, +// H-A-483): each free slot takes the next item through a pull that waits, so an item put or handed back after its pause is +// received the moment the bus gives it; an item handed back reappears only after its pause, so an item put meanwhile goes +// before it. A pull the bus refuses ends the dispatch, said; the holder is pulled again when an item of it is put. +func (pq *publishQueue) dispatch(ctx context.Context, holder string, c jetstream.Consumer) { sem := make(chan struct{}, versions.PublishSendsAtOnce) - run := func(m jetstream.Msg) { - go func() { - defer func() { <-sem }() - pq.take(ctx, m) - }() - } - failed := func(err error) { - pq.logf("the publishes of %s are not pulled: %v; they are pulled when an item of it is put", holder, err) - pq.mu.Lock() - delete(pq.pulling, holder) - pq.mu.Unlock() - } - refused := func(err error) bool { - return err != nil && !errors.Is(err, jetstream.ErrNoMessages) && !errors.Is(err, nats.ErrTimeout) && ctx.Err() == nil - } - // received is the live consumer's highest delivery sequence the dispatch holds; it starts at what the consumer delivered - // before this dispatch, to pulls no longer there. - var received uint64 - if info, err := live.Info(ctx); err != nil { - if refused(err) { - failed(err) - return - } - } else { - received = info.Delivered.Consumer - } - gotLive := func(ms ...jetstream.Msg) { - for _, m := range ms { - if md, err := m.Metadata(); err == nil && md.Sequence.Consumer > received { - received = md.Sequence.Consumer - } - } - } - inFlight := func() bool { // a live item delivered to the waiting pull and not yet received - info, err := live.Info(ctx) - if err != nil { - pq.logf("the publishes of %s: the live consumer cannot be read (%v); the again items wait for the live pull's answer", holder, err) - return true - } - return info.Delivered.Consumer > received - } - now := func(c jetstream.Consumer, n int) ([]jetstream.Msg, error) { - b, err := c.FetchNoWait(n) - if err != nil { - return nil, err - } - var got []jetstream.Msg - for m := range b.Messages() { - got = append(got, m) - } - return got, b.Error() - } - type waited struct { - m jetstream.Msg - err error - } - liveDone, againDone := make(chan waited, 1), make(chan waited, 1) - wait := func(c jetstream.Consumer, done chan<- waited) { - go func() { - var w waited - b, err := c.Fetch(1) - if err == nil { - for m := range b.Messages() { - w.m = m - } - err = b.Error() - } - w.err = err - done <- w - }() - } - var heldLive jetstream.Msg // a live item the waiting pull received - var heldAgain []jetstream.Msg // again items received, not yet run - var liveWaiting, againWaiting bool - heard := func(w waited, waiting *bool, keep func(jetstream.Msg)) bool { - *waiting = false - if refused(w.err) { - failed(w.err) - return false - } - if w.m != nil { - keep(w.m) - } - return true - } - keepLive := func(m jetstream.Msg) { gotLive(m); heldLive = m } - keepAgain := func(m jetstream.Msg) { heldAgain = append(heldAgain, m) } for { select { case sem <- struct{}{}: case <-ctx.Done(): return } - n := 1 - slots: - for n < cap(sem) { - select { - case sem <- struct{}{}: - n++ - default: - break slots + var got jetstream.Msg + b, err := c.Fetch(1) + if err == nil { + for m := range b.Messages() { + got = m } + err = b.Error() } - select { - case w := <-liveDone: - if !heard(w, &liveWaiting, keepLive) { - return - } - default: - } - select { - case w := <-againDone: - if !heard(w, &againWaiting, keepAgain) { - return - } - default: - } - var got []jetstream.Msg - held := false // again items held for a live item in flight - if heldLive != nil { - got, heldLive = []jetstream.Msg{heldLive}, nil - } else { - ms, err := now(live, n) - if refused(err) { - failed(err) - return - } - gotLive(ms...) - got = ms - } - if len(got) == 0 { - if len(heldAgain) == 0 && !againWaiting { - ms, err := now(again, n) - if refused(err) { - failed(err) - return - } - heldAgain = ms - } - if len(heldAgain) > 0 { // with the again items held, live once more: one put since the first fetch goes first - ms, err := now(live, n) - if refused(err) { - failed(err) - return - } - gotLive(ms...) - got = ms - } - switch { - case len(got) > 0 || len(heldAgain) == 0: - case liveWaiting && inFlight(): - held = true - default: - k := min(n, len(heldAgain)) - got, heldAgain = heldAgain[:k], heldAgain[k:] - } - } - for _, m := range got { - run(m) - } - for range n - len(got) { + if err != nil && !errors.Is(err, jetstream.ErrNoMessages) && !errors.Is(err, nats.ErrTimeout) && ctx.Err() == nil { <-sem - } - if len(got) > 0 { - continue - } - if !liveWaiting { - liveWaiting = true - wait(live, liveDone) - } - if !againWaiting && !held { - againWaiting = true - wait(again, againDone) - } - select { - case w := <-liveDone: - if !heard(w, &liveWaiting, keepLive) { - return - } - case w := <-againDone: - if !heard(w, &againWaiting, keepAgain) { - return - } - case <-ctx.Done(): + pq.logf("the publishes of %s are not pulled: %v; they are pulled when an item of it is put", holder, err) + pq.mu.Lock() + delete(pq.pulling, holder) + pq.mu.Unlock() return } + if got == nil { + <-sem + continue + } + go func(m jetstream.Msg) { + defer func() { <-sem }() + pq.take(ctx, m) + }(got) } } diff --git a/modules/mesh-delivery-builder/internal/versions/republish.go b/modules/mesh-delivery-builder/internal/versions/republish.go index 4fa84f819..264371810 100644 --- a/modules/mesh-delivery-builder/internal/versions/republish.go +++ b/modules/mesh-delivery-builder/internal/versions/republish.go @@ -48,7 +48,7 @@ const ( VerbVersions = "versions" // PublishClassLive is the class of a publish whose build ended in this run, which a pipeline's build step waits on; // PublishClassStart that of a publish asked again at a start, of a build that ended before it. A holder's items are pulled - // live first, each class oldest first (design 47 H H-A-290 (1), H2 §793). + // by its one consumer in the order they are put (design 47 H H-A-290 (1), H2 §793, H-A-483). PublishClassLive = "live" PublishClassStart = "again" // PublishSendsAtOnce bounds the sends of one holder that run at once, its consumers' max ack pending (H-A-290 (3), §793).