The builder's publishes have one consumer per holder; publish_first retired (H-A-483) #333
@@ -73,7 +73,7 @@ func publishQueueVia(ask func(key string, body any) (json.RawMessage, error)) fu
|
||||
}
|
||||
}
|
||||
|
||||
// publishActTools are builder_publish_drop and builder_publish_first (design 47 H2 §793): hand acts, recorded in the hand-act log
|
||||
// publishActTools are builder_publish_drop (design 47 H2 §793; builder_publish_first is retired, H-A-483 third addendum): hand acts, recorded in the hand-act log
|
||||
// first, then written to reasks for the builder, which owns the work queue, to perform and answer.
|
||||
func publishActTools(ask queueAsker, now func() time.Time, acts meshhandactsclient.Recorder) []stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
|
||||
@@ -108,7 +108,6 @@ func publishActTools(ask queueAsker, now func() time.Time, acts meshhandactsclie
|
||||
}
|
||||
return []stdio.Tool{
|
||||
tool(versions.PublishDropTool, build.ModePublishDrop, versions.PublishDropDescription, "dropped the publish of"),
|
||||
tool(versions.PublishFirstTool, build.ModePublishFirst, versions.PublishFirstDescription, "put first the publish of"),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -78,8 +78,8 @@ func TestAStateIsReadInOnePage(t *testing.T) {
|
||||
}
|
||||
})
|
||||
// Design 47 H2 §793, H-A-454: builder_publish_queue asks the seat mesh-builds' verb publish-queue, which the builder serves from
|
||||
// its own stream, and answers its counts, list and cap with the states beside them; builder_publish_drop and
|
||||
// builder_publish_first record the hand act first, then write their reasks value.
|
||||
// its own stream, and answers its counts, list and cap with the states beside them; builder_publish_drop records the hand act
|
||||
// first, then writes its reasks value; builder_publish_first is retired (H-A-483, third addendum).
|
||||
t.Run("the queue is the builder's verb publish-queue", func(t *testing.T) {
|
||||
var asked []string
|
||||
ask := func(key string, body any) (json.RawMessage, error) {
|
||||
@@ -123,7 +123,7 @@ func TestAStateIsReadInOnePage(t *testing.T) {
|
||||
t.Errorf("asked %v, answered %+v (%v): want each holder asked in turn at its node, answered together, the silent one with why", asked, hs, err)
|
||||
}
|
||||
})
|
||||
t.Run("drop and first record the hand act, then ask the builder", func(t *testing.T) {
|
||||
t.Run("drop records the hand act, then asks the builder; first is retired", func(t *testing.T) {
|
||||
var order []string
|
||||
ask := func(key string, value []byte) (json.RawMessage, error) {
|
||||
order = append(order, "ask "+key)
|
||||
@@ -141,8 +141,8 @@ func TestAStateIsReadInOnePage(t *testing.T) {
|
||||
t.Fatalf("%s: %v", tool.Name, err)
|
||||
}
|
||||
}
|
||||
if len(tools) != 2 || fmt.Sprint(order) != "[record ask publish-drop.5000000000 record ask publish-first.5000000000]" {
|
||||
t.Errorf("tools %d, order %v: want each hand act recorded before its reasks value", len(tools), order)
|
||||
if len(tools) != 1 || tools[0].Name != versions.PublishDropTool || fmt.Sprint(order) != "[record ask publish-drop.5000000000]" {
|
||||
t.Errorf("tools %d, order %v: want builder_publish_drop alone, its hand act recorded before its reasks value", len(tools), order)
|
||||
}
|
||||
if _, err := tools[0].Run(map[string]any{"module": "m", "why": "no version"}); err == nil {
|
||||
t.Error("a drop naming no version is not refused")
|
||||
|
||||
@@ -273,7 +273,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
Entry: entryVia(calls.Call), Send: sendVia(calls.CallKeptAsk),
|
||||
Unreleased: reserved.unreleasedWords, Logf: logger.Printf,
|
||||
Holders: holders, // a registry seat's holder from the seat-holders view, from the start (design 47 G A-416, A-416a)
|
||||
Started: time.Now()} // a build ended from now on is of the live class, sent first (H-A-290 (1))
|
||||
Started: time.Now()} // a build ended from now on is of the live class (H-A-290 (1))
|
||||
// A number a registry seat already holds is never reserved (H-A-275): read before reserving, through the seats' verb versions.
|
||||
reserved.r.HeldNumbers = func(ctx context.Context, module, fp string, refs []string) ([]string, error) {
|
||||
return republish.HeldNumbers(ctx, module, fp, refs, registryVersionsVia(calls.Call))
|
||||
@@ -328,9 +328,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
acts.PublishDrop = func(ctx context.Context, module, version, by, why string) (string, error) {
|
||||
return pq.drop(ctx, items, module, version, by, why)
|
||||
}
|
||||
acts.PublishFirst = func(ctx context.Context, module, version, _, _ string) (string, error) {
|
||||
return pq.first(ctx, items, module, version)
|
||||
}
|
||||
// builder_reserve_done: a done trunk build that reserved none is given a version, and its publish queued as publish again does.
|
||||
acts.ReserveDone = reserveDoneAct(b.ReserveDone, republish.Republish, logger.Printf)
|
||||
// The operator is asked once a key's event builds are spent (design 47 G §568): the builder's asker on the seat operator-channel.
|
||||
|
||||
+153
-74
@@ -28,15 +28,15 @@ import (
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
|
||||
// Design 47 H2 §793 (whole, against the bus): the builder's work queue publishes, its pull consumers made as it learns a holder.
|
||||
// A live publish enqueued behind 100 again items while the builder was away is sent first; a dropped item is never sent, and
|
||||
// put again within the duplicates window is not stored; the move of the old state run twice puts each item once, and the
|
||||
// builder removes the old state's bucket itself (H-A-461 (1)). The seat mesh-builds' verb publish-queue reads the stream: counts
|
||||
// per holder and class, the oldest and newest, the live item listed first (H-A-454). An idle dispatch sends a live item put
|
||||
// before an again one first, 50 rounds (H-A-473, third addendum). Then §796's tag at another commit, its
|
||||
// reservation's version-reserved waited on by name (§821), and
|
||||
// §794's backlog held six minutes, as steps (H-A-453 (3)).
|
||||
func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) {
|
||||
// Design 47 H2 §793 (whole, against the bus): the builder's work queue publishes, one pull consumer made as it learns a holder
|
||||
// (H-A-483 and its addenda), the class durables of an earlier release retired. No item is lost, a failed send comes back after
|
||||
// its pause behind an item put meanwhile, and a dropped item never comes back; 100 again items and a live one put while the
|
||||
// builder was away are all sent, the dropped one never, and put again within the duplicates window it is not stored; the move
|
||||
// of the old state run twice puts each item once, and the builder removes the old state's bucket itself (H-A-461 (1)). The seat
|
||||
// mesh-builds' verb publish-queue reads the stream: counts per holder and class, the oldest and newest, the items listed in
|
||||
// stream order (H-A-454; H-A-483, third addendum). Then §796's tag at another commit, its reservation's version-reserved waited on by name (§821), and §794's
|
||||
// backlog held six minutes, as steps (H-A-453 (3)).
|
||||
func TestThePublishQueueLosesNoItemAndNeverSendsADroppedItem(t *testing.T) {
|
||||
b := startBed(t)
|
||||
ctx := b.ctx
|
||||
// The stream as the manifest's old streams section declares it: the module's own, retention work-queue, duplicates 24 hours.
|
||||
@@ -45,66 +45,153 @@ func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) {
|
||||
Retention: jetstream.WorkQueuePolicy, Discard: jetstream.DiscardNew, Duplicates: 24 * time.Hour, MaxAge: 7 * 24 * time.Hour}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// An idle dispatch (H-A-473, its third addendum): its two pulls wait; a live item put, then an again one, is sent live first;
|
||||
// and an again item put after a live one, while the dispatch runs an again item put before both, is sent after the live one.
|
||||
// Every time, 50 rounds, each a holder of its own.
|
||||
sentOf := make(chan string, 8)
|
||||
idle := &publishQueue{js: b.js, ackWait: time.Minute, logf: func(string, ...any) {}, binding: publishesOf("mesh-delivery-builder"),
|
||||
take: func(_ context.Context, m itemMsg) {
|
||||
var q versions.QueuedPublish
|
||||
_ = json.Unmarshal(m.Data(), &q)
|
||||
_ = m.Ack()
|
||||
sentOf <- q.Module
|
||||
}}
|
||||
t.Run("idle dispatch", func(t *testing.T) {
|
||||
roundBound := testbus.SetupBound(t) // the level's set-up share (MESH_TEST_LEVEL_BOUND), as the other whole tests
|
||||
for r := range 50 {
|
||||
holder := fmt.Sprintf("idle%d", r)
|
||||
idle.learn(ctx, holder)
|
||||
for _, class := range []string{versions.PublishClassLive, versions.PublishClassStart} { // both pulls stand waiting
|
||||
c, err := b.js.Consumer(ctx, bind.Stream, publishesConsumer(holder, class))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
// One consumer per holder (H-A-483 and its addenda): the holder's class durables of an earlier release are retired, the item
|
||||
// one of them held unacknowledged reaches the one consumer; no item is lost; a failed send comes back after its pause, and a
|
||||
// live item put while it waits is sent before it returns; a dropped item never comes back.
|
||||
t.Run("one consumer", func(t *testing.T) {
|
||||
const holder, pause = "paused", 3 * time.Second // the pause is far longer than a send of an item put now
|
||||
bound := testbus.SetupBound(t)
|
||||
kv := &itemKV{vals: map[string][]byte{}, revs: map[string]uint64{}}
|
||||
type send struct {
|
||||
module string
|
||||
at time.Time
|
||||
}
|
||||
var mu sync.Mutex
|
||||
published, failed := map[string]bool{}, map[string]bool{}
|
||||
sentCh := make(chan send, 16)
|
||||
items := &publishItems{reasks: func() jetstream.KeyValue { return kv }, set: publishSettings{Pause: pause, PauseMax: time.Minute, Tries: 5},
|
||||
now: time.Now, logf: t.Logf,
|
||||
reserved: func(_ context.Context, m, v string) (versions.ReservedVersion, bool, error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return versions.ReservedVersion{Module: m, Version: v, Artifacts: map[string]string{m + ":a": "sha256:a"}, Published: map[string]bool{m + ":a": published[m]}, BuiltAt: time.Now()}, true, nil
|
||||
},
|
||||
send: func(_ context.Context, q versions.QueuedPublish) versions.SendFate {
|
||||
sentCh <- send{q.Module, time.Now()}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if (q.Module == "fail" || q.Module == "dropme") && !failed[q.Module] { // the first send of each fails
|
||||
failed[q.Module] = true
|
||||
return versions.SendFate{Failed: "the registry is down"}
|
||||
}
|
||||
for {
|
||||
info, err := c.Info(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.NumWaiting > 0 {
|
||||
break
|
||||
}
|
||||
published[q.Module] = true
|
||||
return versions.SendFate{Sent: true}
|
||||
}}
|
||||
pq := &publishQueue{js: b.js, ackWait: time.Minute, logf: t.Logf, take: items.take, binding: publishesOf("mesh-delivery-builder")}
|
||||
put := func(module, class string) {
|
||||
q := versions.QueuedPublish{V: 1, Holder: holder, Module: module, Version: "0.1.0", Artifacts: map[string]string{module + ":a": "sha256:a"}, Class: class, Queued: time.Now()}
|
||||
data, _ := json.Marshal(q)
|
||||
if _, err := b.js.Publish(ctx, pq.subject(bind, q), data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
hear := func() { // each publish sent is heard: its item acknowledged once it is held
|
||||
mu.Lock()
|
||||
var done []string
|
||||
for m, ok := range published {
|
||||
if ok {
|
||||
done = append(done, m)
|
||||
}
|
||||
}
|
||||
put := func(module, class string) {
|
||||
q := versions.QueuedPublish{V: 1, Holder: holder, Module: module, Version: "0.1.0", Class: class, Queued: time.Now()}
|
||||
data, _ := json.Marshal(q)
|
||||
if _, err := b.js.Publish(ctx, idle.subject(bind, q), data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mu.Unlock()
|
||||
for _, m := range done {
|
||||
items.heard(ctx, m, "0.1.0")
|
||||
}
|
||||
sent := func(n int) []string {
|
||||
var order []string
|
||||
for range n {
|
||||
select {
|
||||
case m := <-sentOf:
|
||||
order = append(order, m)
|
||||
case <-time.After(roundBound):
|
||||
t.Fatalf("round %d: sent %v of %d items", r, order, n)
|
||||
}
|
||||
}
|
||||
return order
|
||||
}
|
||||
next := func() send {
|
||||
t.Helper()
|
||||
select {
|
||||
case s := <-sentCh:
|
||||
hear()
|
||||
return s
|
||||
case <-time.After(bound):
|
||||
t.Fatal("no item was sent within the level's set-up share")
|
||||
}
|
||||
put("live", versions.PublishClassLive)
|
||||
put("again", versions.PublishClassStart)
|
||||
if order := sent(2); order[0] != "live" {
|
||||
t.Errorf("round %d: an idle dispatch sent %v, want the live item put first sent first", r, order)
|
||||
return send{}
|
||||
}
|
||||
// The two class durables an earlier release made, the live one holding an item given and not acknowledged.
|
||||
for _, class := range []string{versions.PublishClassLive, versions.PublishClassStart} {
|
||||
if _, err := b.js.CreateOrUpdateConsumer(ctx, bind.Stream, jetstream.ConsumerConfig{Durable: publishesConsumer(holder) + "-" + class,
|
||||
FilterSubject: bind.Subject + ".publish." + holder + "." + class + ".>", AckPolicy: jetstream.AckExplicitPolicy, AckWait: time.Hour}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
put("before", versions.PublishClassStart)
|
||||
put("live2", versions.PublishClassLive)
|
||||
put("after", versions.PublishClassStart)
|
||||
if order := sent(3); slices.Index(order, "after") < slices.Index(order, "live2") {
|
||||
t.Errorf("round %d: the dispatch sent %v, want the again item put after the live one sent after it", r, order)
|
||||
}
|
||||
put("given", versions.PublishClassLive)
|
||||
old, err := b.js.Consumer(ctx, bind.Stream, publishesConsumer(holder)+"-"+versions.PublishClassLive)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m, err := old.Next(); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if md, _ := m.Metadata(); md.NumDelivered != 1 {
|
||||
t.Fatalf("the old live durable gave %+v", md)
|
||||
}
|
||||
pq.learn(ctx, holder)
|
||||
stream, err := b.js.Stream(ctx, bind.Stream)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var names []string
|
||||
for n := range stream.ConsumerNames(ctx).Name() {
|
||||
if strings.HasPrefix(n, publishesConsumer(holder)) {
|
||||
names = append(names, n)
|
||||
}
|
||||
}
|
||||
if !slices.Equal(names, []string{publishesConsumer(holder)}) {
|
||||
t.Fatalf("the consumers of %s are %v, want only %s: the class durables retired", holder, names, publishesConsumer(holder))
|
||||
}
|
||||
if s := next(); s.module != "given" {
|
||||
t.Fatalf("sent %s first, want the item the old durable held unacknowledged", s.module)
|
||||
}
|
||||
put("fail", versions.PublishClassStart)
|
||||
put("dropme", versions.PublishClassStart)
|
||||
var firstFail time.Time
|
||||
for range 2 {
|
||||
if s := next(); s.module == "fail" {
|
||||
firstFail = s.at
|
||||
}
|
||||
}
|
||||
put("live", versions.PublishClassLive) // put while both failed items wait out their pause
|
||||
if s := next(); s.module != "live" {
|
||||
t.Fatalf("sent %s, want the live item put while the failed ones wait their pause", s.module)
|
||||
}
|
||||
if answer, err := pq.drop(ctx, items, "dropme", "0.1.0", "op@node", "a test drops it"); err != nil {
|
||||
t.Fatalf("drop: %v", err)
|
||||
} else {
|
||||
t.Log(answer)
|
||||
}
|
||||
if s := next(); s.module != "fail" || s.at.Sub(firstFail) < pause {
|
||||
t.Fatalf("sent %s %s after the failed send, want fail again after its pause %s", s.module, s.at.Sub(firstFail), pause)
|
||||
}
|
||||
// Every item is acknowledged and the dropped one is gone: nothing is left to come back.
|
||||
c, err := b.js.Consumer(ctx, bind.Stream, publishesConsumer(holder))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st, err := b.js.Stream(ctx, bind.Stream)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for deadline := time.Now().Add(bound); ; {
|
||||
hear()
|
||||
info, err := c.Info(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sinfo, err := st.Info(ctx, jetstream.WithSubjectFilter(bind.Subject+".publish."+holder+".>"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.NumAckPending == 0 && info.NumPending == 0 && len(sinfo.State.Subjects) == 0 {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("the queue of %s holds %v, its consumer %d pending and %d unacknowledged: want every item acknowledged and the dropped one gone", holder, sinfo.State.Subjects, info.NumPending, info.NumAckPending)
|
||||
}
|
||||
select {
|
||||
case s := <-sentCh:
|
||||
t.Fatalf("sent %s again, want nothing more", s.module)
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -155,21 +242,13 @@ func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) {
|
||||
raw, err := publishQueueVerb(ctx, pq)("op@node", json.RawMessage(`{}`))
|
||||
if a, ok := raw.(meshbuilds.PublishQueueAnswer); err != nil || !ok || a.Total != 101 || len(a.Counts) != 2 || a.Counts[0].Class != versions.PublishClassStart ||
|
||||
a.Counts[0].Count != 100 || a.Counts[1].Count != 1 || a.Counts[0].Oldest.After(a.Counts[0].Newest) || len(a.Queue) != versions.PublishQueueListCap ||
|
||||
a.Queue[0].Module != "live" || a.Capped {
|
||||
t.Errorf("publish-queue answered %+v (%v), want 101 items, oci's 100 again and 1 live, fifty listed, the live first", raw, err)
|
||||
a.Queue[0].Module != "a000" || a.Capped {
|
||||
t.Errorf("publish-queue answered %+v (%v), want 101 items, oci's 100 again and 1 live, fifty listed in stream order, a000 first", raw, err)
|
||||
}
|
||||
if err := pq.learnAll(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case first := <-sentCh:
|
||||
if first != "live" {
|
||||
t.Errorf("the first item sent is %s, want the live one, enqueued behind 100 again items", first)
|
||||
}
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatal("nothing was sent")
|
||||
}
|
||||
for range 4 { // the again items in flight: at most four held unacknowledged, their publish unheard
|
||||
for range 4 { // the items in flight: at most four held unacknowledged, their publish unheard
|
||||
select {
|
||||
case <-sentCh:
|
||||
case <-time.After(30 * time.Second):
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -531,9 +530,9 @@ func builtOf(all []versions.ReservedVersion) []versions.ReservedVersion {
|
||||
// the manifest's old streams section with the work-queue retention, the module's own, consumed by the module's one process on the
|
||||
// module's account (H-A-464), which declares no bus and so is the stream's one reader by the module's shape; each item on
|
||||
// `mesh.stream.<module>.event.publish.<holder>.<class>.<module>.<version>` under the message id
|
||||
// `<holder>.<class>.<module>.<version>`. The builder makes the pull consumers at run time as it learns a holder,
|
||||
// `publishes-<holder>-<class>`, with ack wait publish-pause-max, max ack pending four and max deliveries ten, and pulls the live
|
||||
// class first, at most four sends at once per holder.
|
||||
// `<holder>.<class>.<module>.<version>`. The builder makes one pull consumer at run time as it learns a holder,
|
||||
// `publishes-<holder>`, with ack wait publish-pause-max, max ack pending four and max deliveries ten, and sends its items in the
|
||||
// order the consumer gives them, at most four at once per holder (H-A-483 and its addenda).
|
||||
type publishQueue struct {
|
||||
js jetstream.JetStream
|
||||
binding func() (publishesStream, error)
|
||||
@@ -555,16 +554,16 @@ func publishesOf(module string) func() (publishesStream, error) {
|
||||
return func() (publishesStream, error) { return b, nil }
|
||||
}
|
||||
|
||||
// The consumers' numbers (H-A-446 (2)): at most four sends in flight per holder and class, and ten deliveries, the work queue's
|
||||
// largest; the builder ends an item at publish-tries anyway.
|
||||
// The consumer's numbers (H-A-446 (2)): at most four sends in flight per holder, and ten deliveries, the work queue's largest;
|
||||
// the builder ends an item at publish-tries anyway.
|
||||
const (
|
||||
publishesMaxAckPending = versions.PublishSendsAtOnce
|
||||
publishesMaxDeliver = 10
|
||||
)
|
||||
|
||||
// publishesConsumer is the durable consumer of holder's items of class.
|
||||
func publishesConsumer(holder, class string) string {
|
||||
return versions.PublishesLocal + "-" + holder + "-" + class
|
||||
// publishesConsumer is the one durable consumer of holder's items (H-A-483 addendum).
|
||||
func publishesConsumer(holder string) string {
|
||||
return versions.PublishesLocal + "-" + holder
|
||||
}
|
||||
|
||||
// subject is the item's subject on the bus.
|
||||
@@ -616,8 +615,15 @@ func (pq *publishQueue) learnAll(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// learn starts holder's pull once.
|
||||
// learn starts holder's pull once. A holder is a module name, so one ending in -live or -again, the class durables an earlier
|
||||
// release made, is refused, said (H-A-483, third addendum): its consumer would be another holder's old durable.
|
||||
func (pq *publishQueue) learn(ctx context.Context, holder string) {
|
||||
for _, class := range []string{versions.PublishClassLive, versions.PublishClassStart} {
|
||||
if strings.HasSuffix(holder, "-"+class) {
|
||||
pq.logf("the publishes of %s are refused: a holder is a module name, and one ending in -%s names the class durable of an earlier release (H-A-483)", holder, class)
|
||||
return
|
||||
}
|
||||
}
|
||||
pq.mu.Lock()
|
||||
if pq.pulling == nil {
|
||||
pq.pulling = map[string]bool{}
|
||||
@@ -636,223 +642,71 @@ func (pq *publishQueue) learn(ctx context.Context, holder string) {
|
||||
}
|
||||
}
|
||||
|
||||
// pull makes holder's two consumers and dispatches their items, the live class first, at most four sends at once: an again item
|
||||
// is run only when no live one waits on the queue or is in flight to the dispatch (dispatch, H-A-473).
|
||||
// pull makes holder's one consumer and dispatches its items (design 47 H2 §793, H-A-483 and its addenda): first it retires the
|
||||
// holder's class durables an earlier release made (`-live`, `-again`), whose filters a work queue refuses beside the one; their
|
||||
// unacknowledged items stay on the queue and reach the one consumer.
|
||||
func (pq *publishQueue) pull(ctx context.Context, holder string) error {
|
||||
b, err := pq.binding()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cons := map[string]jetstream.Consumer{}
|
||||
for _, class := range []string{versions.PublishClassStart, versions.PublishClassLive} {
|
||||
c, err := pq.js.CreateOrUpdateConsumer(ctx, b.Stream, jetstream.ConsumerConfig{Durable: publishesConsumer(holder, class),
|
||||
FilterSubject: b.Subject + ".publish." + versions.EscapeToken(holder) + "." + class + ".>", AckPolicy: jetstream.AckExplicitPolicy,
|
||||
AckWait: pq.ackWait, MaxAckPending: publishesMaxAckPending, MaxDeliver: publishesMaxDeliver})
|
||||
if err != nil {
|
||||
return fmt.Errorf("the consumer %s: %w", publishesConsumer(holder, class), err)
|
||||
for _, class := range []string{versions.PublishClassLive, versions.PublishClassStart} {
|
||||
old := publishesConsumer(holder) + "-" + class
|
||||
err := pq.js.DeleteConsumer(ctx, b.Stream, old)
|
||||
switch {
|
||||
case err == nil:
|
||||
pq.logf("the consumer %s of an earlier release is removed: the publishes of %s have one consumer (H-A-483)", old, holder)
|
||||
case !errors.Is(err, jetstream.ErrConsumerNotFound):
|
||||
return fmt.Errorf("the consumer %s of an earlier release is not removed: %w", old, err)
|
||||
}
|
||||
cons[class] = c
|
||||
}
|
||||
pq.logf("pulling the publishes of %s, the live first", holder)
|
||||
go pq.dispatch(ctx, holder, cons[versions.PublishClassLive], cons[versions.PublishClassStart])
|
||||
c, err := pq.js.CreateOrUpdateConsumer(ctx, b.Stream, jetstream.ConsumerConfig{Durable: publishesConsumer(holder),
|
||||
FilterSubject: b.Subject + ".publish." + versions.EscapeToken(holder) + ".>", AckPolicy: jetstream.AckExplicitPolicy,
|
||||
AckWait: pq.ackWait, MaxAckPending: publishesMaxAckPending, MaxDeliver: publishesMaxDeliver})
|
||||
if err != nil {
|
||||
return fmt.Errorf("the consumer %s: %w", publishesConsumer(holder), err)
|
||||
}
|
||||
pq.logf("pulling the publishes of %s", holder)
|
||||
go pq.dispatch(ctx, holder, c)
|
||||
return nil
|
||||
}
|
||||
|
||||
// dispatch runs the items of one holder, at most four at once, every live item before an again one (design 47 H2 §793,
|
||||
// H-A-473 and its third addendum): on every turn no live item is held it fetches live without waiting, up to its free slots,
|
||||
// and takes again items only when that fetch returned nothing. With again items held it fetches live once more; only when
|
||||
// that returns nothing are they run, and while the live pull waits only once the live consumer's delivered sequence is no
|
||||
// further than the live items received: a gap is a live item in flight to the waiting pull, and the again items wait until it
|
||||
// arrives and runs. When neither holds an item, a pull of one item waits on each, the bus's only wake for an item put or
|
||||
// delivered again. A pull the bus refuses ends the dispatch, said; the holder is pulled again when an item of it is put.
|
||||
func (pq *publishQueue) dispatch(ctx context.Context, holder string, live, again jetstream.Consumer) {
|
||||
// dispatch runs the items of one holder in the order its one consumer gives them, at most four at once (design 47 H2 §793,
|
||||
// H-A-483): each free slot takes the next item through a pull that waits, so an item put or handed back after its pause is
|
||||
// received the moment the bus gives it; an item handed back reappears only after its pause, so an item put meanwhile goes
|
||||
// before it. A pull the bus refuses ends the dispatch, said; the holder is pulled again when an item of it is put.
|
||||
func (pq *publishQueue) dispatch(ctx context.Context, holder string, c jetstream.Consumer) {
|
||||
sem := make(chan struct{}, versions.PublishSendsAtOnce)
|
||||
run := func(m jetstream.Msg) {
|
||||
go func() {
|
||||
defer func() { <-sem }()
|
||||
pq.take(ctx, m)
|
||||
}()
|
||||
}
|
||||
failed := func(err error) {
|
||||
pq.logf("the publishes of %s are not pulled: %v; they are pulled when an item of it is put", holder, err)
|
||||
pq.mu.Lock()
|
||||
delete(pq.pulling, holder)
|
||||
pq.mu.Unlock()
|
||||
}
|
||||
refused := func(err error) bool {
|
||||
return err != nil && !errors.Is(err, jetstream.ErrNoMessages) && !errors.Is(err, nats.ErrTimeout) && ctx.Err() == nil
|
||||
}
|
||||
// received is the live consumer's highest delivery sequence the dispatch holds; it starts at what the consumer delivered
|
||||
// before this dispatch, to pulls no longer there.
|
||||
var received uint64
|
||||
if info, err := live.Info(ctx); err != nil {
|
||||
if refused(err) {
|
||||
failed(err)
|
||||
return
|
||||
}
|
||||
} else {
|
||||
received = info.Delivered.Consumer
|
||||
}
|
||||
gotLive := func(ms ...jetstream.Msg) {
|
||||
for _, m := range ms {
|
||||
if md, err := m.Metadata(); err == nil && md.Sequence.Consumer > received {
|
||||
received = md.Sequence.Consumer
|
||||
}
|
||||
}
|
||||
}
|
||||
inFlight := func() bool { // a live item delivered to the waiting pull and not yet received
|
||||
info, err := live.Info(ctx)
|
||||
if err != nil {
|
||||
pq.logf("the publishes of %s: the live consumer cannot be read (%v); the again items wait for the live pull's answer", holder, err)
|
||||
return true
|
||||
}
|
||||
return info.Delivered.Consumer > received
|
||||
}
|
||||
now := func(c jetstream.Consumer, n int) ([]jetstream.Msg, error) {
|
||||
b, err := c.FetchNoWait(n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var got []jetstream.Msg
|
||||
for m := range b.Messages() {
|
||||
got = append(got, m)
|
||||
}
|
||||
return got, b.Error()
|
||||
}
|
||||
type waited struct {
|
||||
m jetstream.Msg
|
||||
err error
|
||||
}
|
||||
liveDone, againDone := make(chan waited, 1), make(chan waited, 1)
|
||||
wait := func(c jetstream.Consumer, done chan<- waited) {
|
||||
go func() {
|
||||
var w waited
|
||||
b, err := c.Fetch(1)
|
||||
if err == nil {
|
||||
for m := range b.Messages() {
|
||||
w.m = m
|
||||
}
|
||||
err = b.Error()
|
||||
}
|
||||
w.err = err
|
||||
done <- w
|
||||
}()
|
||||
}
|
||||
var heldLive jetstream.Msg // a live item the waiting pull received
|
||||
var heldAgain []jetstream.Msg // again items received, not yet run
|
||||
var liveWaiting, againWaiting bool
|
||||
heard := func(w waited, waiting *bool, keep func(jetstream.Msg)) bool {
|
||||
*waiting = false
|
||||
if refused(w.err) {
|
||||
failed(w.err)
|
||||
return false
|
||||
}
|
||||
if w.m != nil {
|
||||
keep(w.m)
|
||||
}
|
||||
return true
|
||||
}
|
||||
keepLive := func(m jetstream.Msg) { gotLive(m); heldLive = m }
|
||||
keepAgain := func(m jetstream.Msg) { heldAgain = append(heldAgain, m) }
|
||||
for {
|
||||
select {
|
||||
case sem <- struct{}{}:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
n := 1
|
||||
slots:
|
||||
for n < cap(sem) {
|
||||
select {
|
||||
case sem <- struct{}{}:
|
||||
n++
|
||||
default:
|
||||
break slots
|
||||
var got jetstream.Msg
|
||||
b, err := c.Fetch(1)
|
||||
if err == nil {
|
||||
for m := range b.Messages() {
|
||||
got = m
|
||||
}
|
||||
err = b.Error()
|
||||
}
|
||||
select {
|
||||
case w := <-liveDone:
|
||||
if !heard(w, &liveWaiting, keepLive) {
|
||||
return
|
||||
}
|
||||
default:
|
||||
}
|
||||
select {
|
||||
case w := <-againDone:
|
||||
if !heard(w, &againWaiting, keepAgain) {
|
||||
return
|
||||
}
|
||||
default:
|
||||
}
|
||||
var got []jetstream.Msg
|
||||
held := false // again items held for a live item in flight
|
||||
if heldLive != nil {
|
||||
got, heldLive = []jetstream.Msg{heldLive}, nil
|
||||
} else {
|
||||
ms, err := now(live, n)
|
||||
if refused(err) {
|
||||
failed(err)
|
||||
return
|
||||
}
|
||||
gotLive(ms...)
|
||||
got = ms
|
||||
}
|
||||
if len(got) == 0 {
|
||||
if len(heldAgain) == 0 && !againWaiting {
|
||||
ms, err := now(again, n)
|
||||
if refused(err) {
|
||||
failed(err)
|
||||
return
|
||||
}
|
||||
heldAgain = ms
|
||||
}
|
||||
if len(heldAgain) > 0 { // with the again items held, live once more: one put since the first fetch goes first
|
||||
ms, err := now(live, n)
|
||||
if refused(err) {
|
||||
failed(err)
|
||||
return
|
||||
}
|
||||
gotLive(ms...)
|
||||
got = ms
|
||||
}
|
||||
switch {
|
||||
case len(got) > 0 || len(heldAgain) == 0:
|
||||
case liveWaiting && inFlight():
|
||||
held = true
|
||||
default:
|
||||
k := min(n, len(heldAgain))
|
||||
got, heldAgain = heldAgain[:k], heldAgain[k:]
|
||||
}
|
||||
}
|
||||
for _, m := range got {
|
||||
run(m)
|
||||
}
|
||||
for range n - len(got) {
|
||||
if err != nil && !errors.Is(err, jetstream.ErrNoMessages) && !errors.Is(err, nats.ErrTimeout) && ctx.Err() == nil {
|
||||
<-sem
|
||||
}
|
||||
if len(got) > 0 {
|
||||
continue
|
||||
}
|
||||
if !liveWaiting {
|
||||
liveWaiting = true
|
||||
wait(live, liveDone)
|
||||
}
|
||||
if !againWaiting && !held {
|
||||
againWaiting = true
|
||||
wait(again, againDone)
|
||||
}
|
||||
select {
|
||||
case w := <-liveDone:
|
||||
if !heard(w, &liveWaiting, keepLive) {
|
||||
return
|
||||
}
|
||||
case w := <-againDone:
|
||||
if !heard(w, &againWaiting, keepAgain) {
|
||||
return
|
||||
}
|
||||
case <-ctx.Done():
|
||||
pq.logf("the publishes of %s are not pulled: %v; they are pulled when an item of it is put", holder, err)
|
||||
pq.mu.Lock()
|
||||
delete(pq.pulling, holder)
|
||||
pq.mu.Unlock()
|
||||
return
|
||||
}
|
||||
if got == nil {
|
||||
<-sem
|
||||
continue
|
||||
}
|
||||
go func(m jetstream.Msg) {
|
||||
defer func() { <-sem }()
|
||||
pq.take(ctx, m)
|
||||
}(got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -956,45 +810,6 @@ func (pq *publishQueue) drop(ctx context.Context, items *publishItems, module, v
|
||||
return fmt.Sprintf("the publish of %s %s is dropped: %d items terminated, and none is sent or put again", module, version, len(qs)), nil
|
||||
}
|
||||
|
||||
// first is builder_publish_first (§793): each again item of the version is put again as live, under the message id
|
||||
// `<holder>.live.<module>.<version>`, and the old terminated; a live id the queue holds within its window takes nothing, and the
|
||||
// old stays, said.
|
||||
func (pq *publishQueue) first(ctx context.Context, items *publishItems, module, version string) (string, error) {
|
||||
qs, st, err := pq.itemsOf(ctx, module, version, versions.PublishClassStart)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the work queue %s cannot be read: %w", versions.PublishesLocal, err)
|
||||
}
|
||||
if len(qs) == 0 {
|
||||
return "", fmt.Errorf("the work queue %s holds no again item of %s %s", versions.PublishesLocal, module, version)
|
||||
}
|
||||
var moved, kept []string
|
||||
for subj, q := range qs {
|
||||
q.Class = versions.PublishClassLive
|
||||
dup, err := pq.put(ctx, q)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s %s is not put again as live at %s: %w", module, version, q.Holder, err)
|
||||
}
|
||||
if dup {
|
||||
kept = append(kept, q.Holder)
|
||||
continue
|
||||
}
|
||||
if err := st.Purge(ctx, jetstream.WithPurgeSubject(subj)); err != nil {
|
||||
return "", fmt.Errorf("the again item of %s %s at %s is not terminated: %w", module, version, q.Holder, err)
|
||||
}
|
||||
moved = append(moved, q.Holder)
|
||||
}
|
||||
for _, h := range items.settle(module, version, func(h heldItem) bool { return h.q.Class == versions.PublishClassStart }) {
|
||||
_ = h.m.Term()
|
||||
}
|
||||
sort.Strings(moved)
|
||||
answer := fmt.Sprintf("%s %s is put first as live at %s, its again item terminated", module, version, strings.Join(moved, ", "))
|
||||
if len(kept) > 0 {
|
||||
sort.Strings(kept)
|
||||
answer += fmt.Sprintf("; at %s the work queue holds its live id within the duplicates window, so its again item stays", strings.Join(kept, ", "))
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
// endItem writes the version's publish item ended with answer, made when none stands: Try and take then leave it, and a start
|
||||
// asks it no more.
|
||||
func (p *publishItems) endItem(ctx context.Context, module, version, answer string) error {
|
||||
|
||||
@@ -440,3 +440,17 @@ func TestAPublishIsNotAskedAgainWhileItsPublishedMayBeUnheard(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H2 H-A-483 (third addendum): a holder is a module name, so one ending in -live or -again, the class durables of an
|
||||
// earlier release, is refused at the learn, said, and nothing of it is pulled.
|
||||
func TestAHolderNamedLikeAClassDurableIsRefused(t *testing.T) {
|
||||
for _, holder := range []string{"oci-live", "oci-again"} {
|
||||
var said []string
|
||||
pq := &publishQueue{logf: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) },
|
||||
binding: func() (publishesStream, error) { return publishesStream{}, errors.New("no bus in a unit test") }}
|
||||
pq.learn(context.Background(), holder)
|
||||
if len(said) != 1 || !strings.Contains(said[0], "the publishes of "+holder+" are refused") || pq.pulling[holder] {
|
||||
t.Errorf("%s: said %q, pulling %v, want it refused at the learn, said once, nothing pulled", holder, said, pq.pulling[holder])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -65,7 +65,7 @@ func reAsked(ctx context.Context, b askAgainer, acts versions.Acts, items *publi
|
||||
released(ctx, b, kv, e, r, logf)
|
||||
return
|
||||
}
|
||||
if r.Mode == build.ModeRepublish || r.Mode == build.ModePublishDrop || r.Mode == build.ModePublishFirst || r.Mode == build.ModeFloorAgain || r.Mode == build.ModeOldFormDrop || r.Mode == build.ModeReservedDrop || r.Mode == build.ModeReserveDone {
|
||||
if r.Mode == build.ModeRepublish || r.Mode == build.ModePublishDrop || r.Mode == build.ModeFloorAgain || r.Mode == build.ModeOldFormDrop || r.Mode == build.ModeReservedDrop || r.Mode == build.ModeReserveDone {
|
||||
// A version's hand act (design 51 *Tools of the builder*) may build again, so it is answered beside the watch.
|
||||
go versionActed(ctx, acts, kv, e, r, logf)
|
||||
return
|
||||
@@ -208,11 +208,8 @@ func versionActed(ctx context.Context, acts versions.Acts, kv jetstream.KeyValue
|
||||
r.Answer = "queued " + fmt.Sprint(len(r.Queued)) + " versions to publish again: " + strings.Join(r.Queued, ", ") + "; builder_publish_queue shows them"
|
||||
}
|
||||
}
|
||||
} else if r.Mode == build.ModePublishDrop || r.Mode == build.ModePublishFirst {
|
||||
} else if r.Mode == build.ModePublishDrop {
|
||||
act := acts.PublishDrop
|
||||
if r.Mode == build.ModePublishFirst {
|
||||
act = acts.PublishFirst
|
||||
}
|
||||
if act == nil {
|
||||
err = errors.New("this builder keeps no work queue of publishes")
|
||||
} else {
|
||||
|
||||
@@ -74,14 +74,11 @@ const ModePublish = "publish"
|
||||
// optional, name what to publish again; the builder answers with the versions it queued (Queued).
|
||||
const ModeRepublish = "republish"
|
||||
|
||||
// ModePublishDrop and ModePublishFirst are the Modes of a reasks value of builder_publish_drop and builder_publish_first (design
|
||||
// 47 H2 §793): Module and Version name the item; the builder terminates it, or puts it again as live and terminates the old.
|
||||
const (
|
||||
ModePublishDrop = "publish-drop"
|
||||
ModePublishFirst = "publish-first"
|
||||
)
|
||||
// ModePublishDrop is the Mode of a reasks value of builder_publish_drop (design 47 H2 §793): Module and Version name the item;
|
||||
// the builder terminates it.
|
||||
const ModePublishDrop = "publish-drop"
|
||||
|
||||
// NewPublishAct is the value builder_publish_drop (mode ModePublishDrop) or builder_publish_first (ModePublishFirst) writes
|
||||
// NewPublishAct is the value builder_publish_drop (mode ModePublishDrop) writes
|
||||
// (§793): its key, `<mode>.<unix nanoseconds>` (a call is one value), and the value. Module, version and why are required.
|
||||
func NewPublishAct(mode, module, version, why, by string, at time.Time) (string, []byte, error) {
|
||||
module, version = strings.TrimSpace(module), strings.TrimSpace(version)
|
||||
|
||||
@@ -27,10 +27,8 @@ type Acts struct {
|
||||
ReserveDone func(ctx context.Context, module, fp, by, why string, at time.Time) (string, error)
|
||||
// DropReserved drops a module's reservation of one MAJOR.MINOR by hand (builder_reserved_drop): its answer, or why not.
|
||||
DropReserved func(ctx context.Context, module, series, by, why string, at time.Time) (string, error)
|
||||
// PublishDrop terminates one publish item, and PublishFirst puts it again as live and terminates the old (builder_publish_drop,
|
||||
// builder_publish_first, design 47 H2 §793): the answer, or why not.
|
||||
PublishDrop func(ctx context.Context, module, version, by, why string) (string, error)
|
||||
PublishFirst func(ctx context.Context, module, version, by, why string) (string, error)
|
||||
// PublishDrop terminates one publish item (builder_publish_drop, design 47 H2 §793): the answer, or why not.
|
||||
PublishDrop func(ctx context.Context, module, version, by, why string) (string, error)
|
||||
}
|
||||
|
||||
// FloorAgain is builder_floor_again of name: refused, since the builder numbers nothing of its own and no claim asks a floor
|
||||
|
||||
@@ -37,9 +37,8 @@ const (
|
||||
PublishQueueTool = "builder_publish_queue"
|
||||
// QueueFull is what version-unreleased says of a version the work queue's cap refused.
|
||||
QueueFull = "queue full"
|
||||
// PublishDropTool and PublishFirstTool are builder_publish_drop and builder_publish_first (§793).
|
||||
PublishDropTool = "builder_publish_drop"
|
||||
PublishFirstTool = "builder_publish_first"
|
||||
// PublishDropTool is builder_publish_drop (§793); builder_publish_first is retired (H-A-483, third addendum).
|
||||
PublishDropTool = "builder_publish_drop"
|
||||
// NotStoredWords opens what a refused artifact is said as.
|
||||
NotStoredWords = "not stored: "
|
||||
// VerbPublish is the registry seats' verb a publish again sends, VerbVersions the one whose record says what a seat holds
|
||||
@@ -48,7 +47,7 @@ const (
|
||||
VerbVersions = "versions"
|
||||
// PublishClassLive is the class of a publish whose build ended in this run, which a pipeline's build step waits on;
|
||||
// PublishClassStart that of a publish asked again at a start, of a build that ended before it. A holder's items are pulled
|
||||
// live first, each class oldest first (design 47 H H-A-290 (1), H2 §793).
|
||||
// by its one consumer in the order they are put (design 47 H H-A-290 (1), H2 §793, H-A-483).
|
||||
PublishClassLive = "live"
|
||||
PublishClassStart = "again"
|
||||
// PublishSendsAtOnce bounds the sends of one holder that run at once, its consumers' max ack pending (H-A-290 (3), §793).
|
||||
@@ -97,21 +96,19 @@ type QueuedPublish struct {
|
||||
Class string `json:"class,omitempty"`
|
||||
}
|
||||
|
||||
// rank is the place of q's class in a holder's queue: the live first.
|
||||
func (q QueuedPublish) rank() int {
|
||||
if q.Class == PublishClassLive {
|
||||
return 0
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
// sortQueued sorts a holder's queued versions in the order they are sent: the live class first, each class oldest first.
|
||||
func sortQueued(qs []QueuedPublish) {
|
||||
sort.SliceStable(qs, func(i, j int) bool {
|
||||
if a, b := qs[i].rank(), qs[j].rank(); a != b {
|
||||
// streamOrder sorts the keys of the work queue's items in stream order: each key is its item's stream sequence (the builder's
|
||||
// reader, publishitem.go), compared as a number; a key that is no number sorts after them by its words.
|
||||
func streamOrder(keys []string) {
|
||||
sort.SliceStable(keys, func(i, j int) bool {
|
||||
a, aerr := strconv.ParseUint(keys[i], 10, 64)
|
||||
b, berr := strconv.ParseUint(keys[j], 10, 64)
|
||||
switch {
|
||||
case aerr == nil && berr == nil:
|
||||
return a < b
|
||||
case aerr == nil || berr == nil:
|
||||
return aerr == nil
|
||||
}
|
||||
return qs[i].Queued.Before(qs[j].Queued)
|
||||
return keys[i] < keys[j]
|
||||
})
|
||||
}
|
||||
|
||||
@@ -456,7 +453,7 @@ func (r *Republisher) Republish(ctx context.Context, module, version string) ([]
|
||||
type Queued struct {
|
||||
Total int `json:"total"`
|
||||
Counts []QueueCount `json:"counts"`
|
||||
// Queue are the first queued versions, per holder in the order they are sent (the live class first, each class oldest first);
|
||||
// Queue are the first queued versions, in stream order, the order each holder's one consumer gives them (H-A-483);
|
||||
// with a module named, only that module's.
|
||||
Queue []QueuedPublish `json:"queue"`
|
||||
// Listed says how many of Total the answer lists.
|
||||
@@ -512,7 +509,8 @@ func PublishItemKey(module, version string) string {
|
||||
|
||||
// PublishQueueOf is the seat mesh-builds' verb publish-queue (design 47 H2 §793; H-A-454): the builder's read of its own work queue
|
||||
// publishes (queue, its items read in bulk, capped past its bound), small at any size of the queue (H-A-290 (5)): the items' count
|
||||
// per holder and class with the oldest and newest queue time, and the first PublishQueueListCap in the order they are sent; module
|
||||
// per holder and class with the oldest and newest queue time, and the first PublishQueueListCap in stream order, the order the
|
||||
// holder's one consumer gives them (H-A-483, third addendum); module
|
||||
// (and version) narrow the list to one module's (one version's), so an item's class can be read.
|
||||
func PublishQueueOf(queue Reader, module, version string) (meshbuilds.PublishQueueAnswer, error) {
|
||||
out := meshbuilds.PublishQueueAnswer{Counts: []meshbuilds.PublishQueueAnswerCountsItem{}, Queue: []meshbuilds.PublishQueueAnswerQueueItem{}}
|
||||
@@ -525,7 +523,13 @@ func PublishQueueOf(queue Reader, module, version string) (meshbuilds.PublishQue
|
||||
}
|
||||
counts := map[[2]string]*meshbuilds.PublishQueueAnswerCountsItem{}
|
||||
var listed []QueuedPublish
|
||||
for _, raw := range vals {
|
||||
keys := make([]string, 0, len(vals))
|
||||
for k := range vals {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
streamOrder(keys)
|
||||
for _, k := range keys {
|
||||
raw := vals[k]
|
||||
var q QueuedPublish
|
||||
if json.Unmarshal(raw, &q) != nil {
|
||||
continue
|
||||
@@ -559,25 +563,12 @@ func PublishQueueOf(queue Reader, module, version string) (meshbuilds.PublishQue
|
||||
}
|
||||
return out.Counts[i].Class < out.Counts[j].Class
|
||||
})
|
||||
sort.SliceStable(listed, func(i, j int) bool { return listed[i].Holder < listed[j].Holder })
|
||||
byHolder := map[string][]QueuedPublish{}
|
||||
var holders []string
|
||||
for _, q := range listed {
|
||||
if byHolder[q.Holder] == nil {
|
||||
holders = append(holders, q.Holder)
|
||||
}
|
||||
byHolder[q.Holder] = append(byHolder[q.Holder], q)
|
||||
}
|
||||
for _, h := range holders {
|
||||
qs := byHolder[h]
|
||||
sortQueued(qs)
|
||||
for _, q := range qs {
|
||||
if len(out.Queue) == PublishQueueListCap {
|
||||
break
|
||||
}
|
||||
out.Queue = append(out.Queue, meshbuilds.PublishQueueAnswerQueueItem{Holder: q.Holder, Module: q.Module, Version: q.Version,
|
||||
Class: q.Class, Artifacts: q.Artifacts, Queued: q.Queued})
|
||||
if len(out.Queue) == PublishQueueListCap {
|
||||
break
|
||||
}
|
||||
out.Queue = append(out.Queue, meshbuilds.PublishQueueAnswerQueueItem{Holder: q.Holder, Module: q.Module, Version: q.Version,
|
||||
Class: q.Class, Artifacts: q.Artifacts, Queued: q.Queued})
|
||||
}
|
||||
out.Listed = int64(len(out.Queue))
|
||||
return out, nil
|
||||
@@ -631,20 +622,17 @@ func AnswerQueue(q meshbuilds.PublishQueueAnswer, acted, reasks Reader) (Queued,
|
||||
|
||||
// PublishQueueDescription is builder_publish_queue's description.
|
||||
const PublishQueueDescription = "The versions the builder will publish again (its work queue " + PublishesLocal + ", design 47 G section 588, H2 section 793): " +
|
||||
"how many each registry seat holder's items hold per class (live, a build a pipeline's step waits on, pulled first, or again, asked again at a start) " +
|
||||
"with the oldest and newest queue time; the first 50 in the order they are pulled, each with its class, its artifacts' digests and when it was " +
|
||||
"how many each registry seat holder's items hold per class (live, a build a pipeline's step waits on, or again, asked again at a start) " +
|
||||
"with the oldest and newest queue time; the first 50 in stream order, the order each holder's one consumer pulls them, each with its class, its artifacts' digests and when it was " +
|
||||
"queued (module, and version, list only that module's, so one item's class can be read); the keys of the state " + ActedState + ", the last stream " +
|
||||
"sequence the builder acted on for each subject; and the publish items (design 47 G A-457a), counted, the first 50 by next try, each version's " +
|
||||
"tries and next try, and why one ended (its tries spent, or dropped). A read past its pages is answered capped, naming the items read and the " +
|
||||
"queue's size (H-A-293). Read-only."
|
||||
|
||||
// PublishDropDescription and PublishFirstDescription are builder_publish_drop's and builder_publish_first's (§793).
|
||||
// PublishDropDescription is builder_publish_drop's (§793).
|
||||
const PublishDropDescription = "Drop one version's publish (design 47 H2 section 793): its items on the work queue " + PublishesLocal + " are " +
|
||||
"terminated at every holder, and its publish item ended with who dropped it and why, so it is never sent or put again; builder_publish_again " +
|
||||
"asks it anew. A hand act, recorded in the hand-act log first."
|
||||
const PublishFirstDescription = "Put one version's publish first (design 47 H2 section 793): each of its again items on the work queue " +
|
||||
PublishesLocal + " is put again as live, under the message id <holder>.live.<module>.<version>, pulled before every again item, and the old " +
|
||||
"terminated. A hand act, recorded in the hand-act log first."
|
||||
|
||||
// PublishAgainDescription is builder_publish_again's description.
|
||||
const PublishAgainDescription = "Publish reserved versions again (design 47 G section 588), which builds nothing: with module and version, that reserved " +
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -404,8 +405,9 @@ func answerOf(queue, acted, reasks Reader, module, version string) (Queued, erro
|
||||
}
|
||||
|
||||
// TestThePublishQueueToolAnswersAtAnySize: design 47 H H-A-290 (5) — builder_publish_queue over two thousand queued versions answers
|
||||
// within a second and small: counts per holder and class with the oldest and newest queue time, a list capped at fifty in the
|
||||
// order they are sent, and one module's item with its class; each state read in bulk where the reader can.
|
||||
// within a second and small: counts per holder and class with the oldest and newest queue time, a list capped at fifty in stream
|
||||
// order (H-A-483, third addendum: each key is its item's stream sequence), and one module's item with its class; each state read
|
||||
// in bulk where the reader can.
|
||||
func TestThePublishQueueToolAnswersAtAnySize(t *testing.T) {
|
||||
at := time.Date(2026, 11, 20, 17, 0, 0, 0, time.UTC)
|
||||
vals := map[string]json.RawMessage{}
|
||||
@@ -418,7 +420,7 @@ func TestThePublishQueueToolAnswersAtAnySize(t *testing.T) {
|
||||
class = PublishClassLive
|
||||
}
|
||||
q := QueuedPublish{V: 1, Holder: holder, Module: fmt.Sprintf("m%d", i), Version: "0.1.0", Artifacts: map[string]string{"a": "sha256:aa"}, Queued: at.Add(time.Duration(i) * time.Second), Class: class}
|
||||
vals[QueueKey(holder, q.Module, q.Version)], _ = json.Marshal(q)
|
||||
vals[strconv.Itoa(i+1)], _ = json.Marshal(q) // the reader keys each item by its stream sequence
|
||||
}
|
||||
gets := 0
|
||||
queue := Reader{All: func() (StatePage, error) { return StatePage{Values: vals}, nil },
|
||||
@@ -435,8 +437,10 @@ func TestThePublishQueueToolAnswersAtAnySize(t *testing.T) {
|
||||
if fmt.Sprintln(a.Counts[1].Holder, a.Counts[1].Class, a.Counts[1].Count, a.Counts[1].Oldest.Equal(at.Add(1991*time.Second)), a.Counts[1].Newest.Equal(at.Add(1999*time.Second))) != "go "+PublishClassLive+" 5 true true\n" || len(a.Counts) != 4 {
|
||||
t.Errorf("counts %+v, want per holder and class with their oldest and newest", a.Counts)
|
||||
}
|
||||
if a.Queue[0].Class != PublishClassLive || a.Queue[0].Holder != "go" {
|
||||
t.Errorf("the list opens with %+v, want the go holder's live class, sent first", a.Queue[0])
|
||||
for i, q := range a.Queue {
|
||||
if q.Module != fmt.Sprintf("m%d", i) {
|
||||
t.Fatalf("the list holds %s at %d, want m%d: the items in stream order, whatever their holder or class", q.Module, i, i)
|
||||
}
|
||||
}
|
||||
one, err := answerOf(queue, none, none, "m1995", "")
|
||||
if err != nil || len(one.Queue) != 1 || one.Queue[0].Class != PublishClassLive || one.Total != 2000 {
|
||||
|
||||
@@ -57,7 +57,7 @@ func VerbDescription(verb string) string {
|
||||
case VerbDrop:
|
||||
return "End an ask on the seat's work queue for a caller that no longer waits for it (its step ended by its bound): one waiting is ended dropped when it is pulled, one taken is cancelled; both end the step could-not-run, its why naming the drop and who asked it, so no holder works on it after (design 47 H2 §778; H-A-401; ADR 0664 §4)."
|
||||
case VerbPublishQueue:
|
||||
return "What the builder's work queue publishes holds, read by the builder from its own stream (design 47 H2 §793; H-A-454): the items' count per registry seat holder and class with the oldest and newest queue time, and the first 50 in the order they are pulled (the live class first, each class oldest first); module, and version, list only that module's items. A queue read past 4,000 items is answered capped, naming the items read and the queue's size."
|
||||
return "What the builder's work queue publishes holds, read by the builder from its own stream (design 47 H2 §793; H-A-454): the items' count per registry seat holder and class with the oldest and newest queue time, and the first 50 in stream order, the order each holder's one consumer pulls them (H-A-483); module, and version, list only that module's items. A queue read past 4,000 items is answered capped, naming the items read and the queue's size."
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -338,7 +338,7 @@ const (
|
||||
type PublishQueueAnswerQueueItem struct {
|
||||
// each artifact reference, <module>:<artifact>, with the digest to publish
|
||||
Artifacts map[string]string `json:"artifacts,omitempty"`
|
||||
// live, a build a pipeline's step waits on, pulled first; again, asked again at a start
|
||||
// live, a build a pipeline's step waits on; again, asked again at a start
|
||||
Class string `json:"class"`
|
||||
// the registry seat's holder the item is sent to, a module
|
||||
Holder string `json:"holder"`
|
||||
@@ -368,7 +368,7 @@ type PublishQueueAnswer struct {
|
||||
ItemsRead int64 `json:"items-read,omitempty"`
|
||||
// how many of total the answer lists
|
||||
Listed int64 `json:"listed"`
|
||||
// the first 50 items listed, per holder in the order they are pulled
|
||||
// the first 50 items listed, in stream order
|
||||
Queue []PublishQueueAnswerQueueItem `json:"queue"`
|
||||
// with capped: the queue's size in bytes
|
||||
QueueBytes int64 `json:"queue-bytes,omitempty"`
|
||||
|
||||
@@ -156,7 +156,7 @@
|
||||
{
|
||||
"name": "publish-queue",
|
||||
"class": "read",
|
||||
"description": "What the builder's work queue publishes holds, read by the builder from its own stream (design 47 H2 §793; H-A-454): the items' count per registry seat holder and class with the oldest and newest queue time, and the first 50 in the order they are pulled (the live class first, each class oldest first); module, and version, list only that module's items. A queue read past 4,000 items is answered capped, naming the items read and the queue's size.",
|
||||
"description": "What the builder's work queue publishes holds, read by the builder from its own stream (design 47 H2 §793; H-A-454): the items' count per registry seat holder and class with the oldest and newest queue time, and the first 50 in stream order, the order each holder's one consumer pulls them (H-A-483); module, and version, list only that module's items. A queue read past 4,000 items is answered capped, naming the items read and the queue's size.",
|
||||
"input": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -243,7 +243,7 @@
|
||||
"live",
|
||||
"again"
|
||||
],
|
||||
"description": "live, a build a pipeline's step waits on, pulled first; again, asked again at a start"
|
||||
"description": "live, a build a pipeline's step waits on; again, asked again at a start"
|
||||
},
|
||||
"artifacts": {
|
||||
"type": "object",
|
||||
@@ -266,7 +266,7 @@
|
||||
"queued"
|
||||
]
|
||||
},
|
||||
"description": "the first 50 items listed, per holder in the order they are pulled"
|
||||
"description": "the first 50 items listed, in stream order"
|
||||
},
|
||||
"listed": {
|
||||
"type": "integer",
|
||||
|
||||
@@ -251,7 +251,7 @@ export interface PublishQueueAnswerCountsItem {
|
||||
export interface PublishQueueAnswerQueueItem {
|
||||
/** each artifact reference, <module>:<artifact>, with the digest to publish */
|
||||
"artifacts"?: Record<string, string>;
|
||||
/** live, a build a pipeline's step waits on, pulled first; again, asked again at a start */
|
||||
/** live, a build a pipeline's step waits on; again, asked again at a start */
|
||||
"class": "live" | "again";
|
||||
/** the registry seat's holder the item is sent to, a module */
|
||||
"holder": string;
|
||||
@@ -275,7 +275,7 @@ export interface PublishQueueAnswer {
|
||||
"items-read"?: number;
|
||||
/** how many of total the answer lists */
|
||||
"listed": number;
|
||||
/** the first 50 items listed, per holder in the order they are pulled */
|
||||
/** the first 50 items listed, in stream order */
|
||||
"queue": PublishQueueAnswerQueueItem[];
|
||||
/** with capped: the queue's size in bytes */
|
||||
"queue-bytes"?: number;
|
||||
|
||||
Reference in New Issue
Block a user