diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go index e05709ad0..f10fd3886 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go @@ -45,98 +45,6 @@ func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) { Retention: jetstream.WorkQueuePolicy, Discard: jetstream.DiscardNew, Duplicates: 24 * time.Hour, MaxAge: 7 * 24 * time.Hour}); err != nil { t.Fatal(err) } - kv := &itemKV{vals: map[string][]byte{}, revs: map[string]uint64{}} - var mu sync.Mutex - var sent []string - published := map[string]bool{} // the modules whose publish is heard - sentCh := make(chan string, 200) - items := &publishItems{reasks: func() jetstream.KeyValue { return kv }, set: publishSettings{Pause: time.Second, PauseMax: time.Minute, Tries: 5}, - now: time.Now, logf: t.Logf, - reserved: func(_ context.Context, m, v string) (versions.ReservedVersion, bool, error) { - mu.Lock() - defer mu.Unlock() - return versions.ReservedVersion{Module: m, Version: v, Artifacts: map[string]string{m + ":a": "sha256:a"}, Published: map[string]bool{m + ":a": published[m]}, BuiltAt: time.Now()}, true, nil - }, - send: func(_ context.Context, q versions.QueuedPublish) versions.SendFate { - mu.Lock() - sent = append(sent, q.Module) - mu.Unlock() - sentCh <- q.Module - return versions.SendFate{Sent: true} - }} - pq := &publishQueue{js: b.js, ackWait: time.Minute, logf: t.Logf, take: items.take, binding: publishesOf("mesh-delivery-builder")} - item := func(module, class string) versions.QueuedPublish { - return versions.QueuedPublish{V: 1, Holder: "oci", Module: module, Version: "0.1.0", Artifacts: map[string]string{module + ":a": "sha256:a"}, Class: class, Queued: time.Now()} - } - away := func(q versions.QueuedPublish) { // put while the builder was away: no holder learnt, nothing pulled - data, _ := json.Marshal(q) - if _, err := b.js.Publish(ctx, pq.subject(bind, q), data, jetstream.WithMsgID(versions.PublishMsgID(q.Holder, q.Class, q.Module, q.Version))); err != nil { - t.Fatal(err) - } - } - for i := range 100 { - away(item(fmt.Sprintf("a%03d", i), versions.PublishClassStart)) - } - away(item("dropme", versions.PublishClassStart)) - away(item("live", versions.PublishClassLive)) - if answer, err := pq.drop(ctx, items, "dropme", "0.1.0", "op@node", "a test drops it"); err != nil { - t.Fatalf("drop: %v", err) - } else { - t.Log(answer) - } - if dup, err := pq.put(ctx, item("dropme", versions.PublishClassStart)); err != nil || !dup { - t.Errorf("a dropped item put again within the window: duplicate %v (%v), want it not stored", dup, err) - } - raw, err := publishQueueVerb(ctx, pq)("op@node", json.RawMessage(`{}`)) - if a, ok := raw.(meshbuilds.PublishQueueAnswer); err != nil || !ok || a.Total != 101 || len(a.Counts) != 2 || a.Counts[0].Class != versions.PublishClassStart || - a.Counts[0].Count != 100 || a.Counts[1].Count != 1 || a.Counts[0].Oldest.After(a.Counts[0].Newest) || len(a.Queue) != versions.PublishQueueListCap || - a.Queue[0].Module != "live" || a.Capped { - t.Errorf("publish-queue answered %+v (%v), want 101 items, oci's 100 again and 1 live, fifty listed, the live first", raw, err) - } - if err := pq.learnAll(ctx); err != nil { - t.Fatal(err) - } - select { - case first := <-sentCh: - if first != "live" { - t.Errorf("the first item sent is %s, want the live one, enqueued behind 100 again items", first) - } - case <-time.After(30 * time.Second): - t.Fatal("nothing was sent") - } - for range 4 { // the again items in flight: at most four held unacknowledged, their publish unheard - select { - case <-sentCh: - case <-time.After(30 * time.Second): - t.Fatal("the again items were not sent") - } - } - deadline := time.After(60 * time.Second) - for { // each publish sent is heard: its item acknowledged, the next sent - mu.Lock() - n, done := len(sent), slices.Clone(sent) - for _, m := range done { - published[m] = true - } - mu.Unlock() - if n >= 101 { - break - } - for _, m := range done { - items.heard(ctx, m, "0.1.0") - } - select { - case <-sentCh: - case <-deadline: - t.Fatalf("sent %d of 101", n) - } - } - mu.Lock() - if slices.Contains(sent, "dropme") || len(sent) != 101 { - t.Errorf("sent %d items (dropme among them: %v), want the 101 and never the dropped one", len(sent), slices.Contains(sent, "dropme")) - } - mu.Unlock() - // An idle dispatch (H-A-473, its third addendum): its two pulls wait; a live item put, then an again one, is sent live first; // and an again item put after a live one, while the dispatch runs an again item put before both, is sent after the live one. // Every time, 50 rounds, each a holder of its own. @@ -148,240 +56,336 @@ func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) { _ = m.Ack() sentOf <- q.Module }} - roundBound := testbus.SetupBound(t) // the level's set-up share (MESH_TEST_LEVEL_BOUND), as the other whole tests - for r := range 50 { - holder := fmt.Sprintf("idle%d", r) - idle.learn(ctx, holder) - for _, class := range []string{versions.PublishClassLive, versions.PublishClassStart} { // both pulls stand waiting - c, err := b.js.Consumer(ctx, bind.Stream, publishesConsumer(holder, class)) - if err != nil { - t.Fatal(err) - } - for { - info, err := c.Info(ctx) + t.Run("idle dispatch", func(t *testing.T) { + roundBound := testbus.SetupBound(t) // the level's set-up share (MESH_TEST_LEVEL_BOUND), as the other whole tests + for r := range 50 { + holder := fmt.Sprintf("idle%d", r) + idle.learn(ctx, holder) + for _, class := range []string{versions.PublishClassLive, versions.PublishClassStart} { // both pulls stand waiting + c, err := b.js.Consumer(ctx, bind.Stream, publishesConsumer(holder, class)) if err != nil { t.Fatal(err) } - if info.NumWaiting > 0 { - break + for { + info, err := c.Info(ctx) + if err != nil { + t.Fatal(err) + } + if info.NumWaiting > 0 { + break + } } } + put := func(module, class string) { + q := versions.QueuedPublish{V: 1, Holder: holder, Module: module, Version: "0.1.0", Class: class, Queued: time.Now()} + data, _ := json.Marshal(q) + if _, err := b.js.Publish(ctx, idle.subject(bind, q), data); err != nil { + t.Fatal(err) + } + } + sent := func(n int) []string { + var order []string + for range n { + select { + case m := <-sentOf: + order = append(order, m) + case <-time.After(roundBound): + t.Fatalf("round %d: sent %v of %d items", r, order, n) + } + } + return order + } + put("live", versions.PublishClassLive) + put("again", versions.PublishClassStart) + if order := sent(2); order[0] != "live" { + t.Errorf("round %d: an idle dispatch sent %v, want the live item put first sent first", r, order) + } + put("before", versions.PublishClassStart) + put("live2", versions.PublishClassLive) + put("after", versions.PublishClassStart) + if order := sent(3); slices.Index(order, "after") < slices.Index(order, "live2") { + t.Errorf("round %d: the dispatch sent %v, want the again item put after the live one sent after it", r, order) + } } - put := func(module, class string) { - q := versions.QueuedPublish{V: 1, Holder: holder, Module: module, Version: "0.1.0", Class: class, Queued: time.Now()} + }) + + t.Run("the queue, the move, tags and backlog", func(t *testing.T) { + kv := &itemKV{vals: map[string][]byte{}, revs: map[string]uint64{}} + var mu sync.Mutex + var sent []string + published := map[string]bool{} // the modules whose publish is heard + sentCh := make(chan string, 200) + items := &publishItems{reasks: func() jetstream.KeyValue { return kv }, set: publishSettings{Pause: time.Second, PauseMax: time.Minute, Tries: 5}, + now: time.Now, logf: t.Logf, + reserved: func(_ context.Context, m, v string) (versions.ReservedVersion, bool, error) { + mu.Lock() + defer mu.Unlock() + return versions.ReservedVersion{Module: m, Version: v, Artifacts: map[string]string{m + ":a": "sha256:a"}, Published: map[string]bool{m + ":a": published[m]}, BuiltAt: time.Now()}, true, nil + }, + send: func(_ context.Context, q versions.QueuedPublish) versions.SendFate { + mu.Lock() + sent = append(sent, q.Module) + mu.Unlock() + sentCh <- q.Module + return versions.SendFate{Sent: true} + }} + pq := &publishQueue{js: b.js, ackWait: time.Minute, logf: t.Logf, take: items.take, binding: publishesOf("mesh-delivery-builder")} + item := func(module, class string) versions.QueuedPublish { + return versions.QueuedPublish{V: 1, Holder: "oci", Module: module, Version: "0.1.0", Artifacts: map[string]string{module + ":a": "sha256:a"}, Class: class, Queued: time.Now()} + } + away := func(q versions.QueuedPublish) { // put while the builder was away: no holder learnt, nothing pulled data, _ := json.Marshal(q) - if _, err := b.js.Publish(ctx, idle.subject(bind, q), data); err != nil { + if _, err := b.js.Publish(ctx, pq.subject(bind, q), data, jetstream.WithMsgID(versions.PublishMsgID(q.Holder, q.Class, q.Module, q.Version))); err != nil { t.Fatal(err) } } - sent := func(n int) []string { - var order []string - for range n { - select { - case m := <-sentOf: - order = append(order, m) - case <-time.After(roundBound): - t.Fatalf("round %d: sent %v of %d items", r, order, n) - } + for i := range 100 { + away(item(fmt.Sprintf("a%03d", i), versions.PublishClassStart)) + } + away(item("dropme", versions.PublishClassStart)) + away(item("live", versions.PublishClassLive)) + if answer, err := pq.drop(ctx, items, "dropme", "0.1.0", "op@node", "a test drops it"); err != nil { + t.Fatalf("drop: %v", err) + } else { + t.Log(answer) + } + if dup, err := pq.put(ctx, item("dropme", versions.PublishClassStart)); err != nil || !dup { + t.Errorf("a dropped item put again within the window: duplicate %v (%v), want it not stored", dup, err) + } + raw, err := publishQueueVerb(ctx, pq)("op@node", json.RawMessage(`{}`)) + if a, ok := raw.(meshbuilds.PublishQueueAnswer); err != nil || !ok || a.Total != 101 || len(a.Counts) != 2 || a.Counts[0].Class != versions.PublishClassStart || + a.Counts[0].Count != 100 || a.Counts[1].Count != 1 || a.Counts[0].Oldest.After(a.Counts[0].Newest) || len(a.Queue) != versions.PublishQueueListCap || + a.Queue[0].Module != "live" || a.Capped { + t.Errorf("publish-queue answered %+v (%v), want 101 items, oci's 100 again and 1 live, fifty listed, the live first", raw, err) + } + if err := pq.learnAll(ctx); err != nil { + t.Fatal(err) + } + select { + case first := <-sentCh: + if first != "live" { + t.Errorf("the first item sent is %s, want the live one, enqueued behind 100 again items", first) } - return order + case <-time.After(30 * time.Second): + t.Fatal("nothing was sent") } - put("live", versions.PublishClassLive) - put("again", versions.PublishClassStart) - if order := sent(2); order[0] != "live" { - t.Errorf("round %d: an idle dispatch sent %v, want the live item put first sent first", r, order) + for range 4 { // the again items in flight: at most four held unacknowledged, their publish unheard + select { + case <-sentCh: + case <-time.After(30 * time.Second): + t.Fatal("the again items were not sent") + } } - put("before", versions.PublishClassStart) - put("live2", versions.PublishClassLive) - put("after", versions.PublishClassStart) - if order := sent(3); slices.Index(order, "after") < slices.Index(order, "live2") { - t.Errorf("round %d: the dispatch sent %v, want the again item put after the live one sent after it", r, order) + deadline := time.After(60 * time.Second) + for { // each publish sent is heard: its item acknowledged, the next sent + mu.Lock() + n, done := len(sent), slices.Clone(sent) + for _, m := range done { + published[m] = true + } + mu.Unlock() + if n >= 101 { + break + } + for _, m := range done { + items.heard(ctx, m, "0.1.0") + } + select { + case <-sentCh: + case <-deadline: + t.Fatalf("sent %d of 101", n) + } } - } + mu.Lock() + if slices.Contains(sent, "dropme") || len(sent) != 101 { + t.Errorf("sent %d items (dropme among them: %v), want the 101 and never the dropped one", len(sent), slices.Contains(sent, "dropme")) + } + mu.Unlock() - // The move of the old state, run twice, puts each item once (at another holder, so the pull of oci is not asked). - old, err := b.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: builds.StateBucket("mesh-delivery-builder", versions.PublishQueueState)}) - if err != nil { - t.Fatal(err) - } - var tagsWritten []string - call := func(tool string, body any) (json.RawMessage, error) { - switch tool { - case "seat:" + gitseat.Seat + "." + gitseat.VerbTags: // m4's 0.2.0 is tagged at another commit (§796) - return json.Marshal(gitseat.TagsAnswer{Form: 2, Tags: []gitseat.Tag{{Tag: "modules/m4/v0.2.0", Commit: "c-other"}}}) - case "seat:" + gitseat.Seat + ".tag": - tagsWritten = append(tagsWritten, body.(gitseat.TagInput).Tag+"@"+body.(gitseat.TagInput).Commit) - return json.Marshal(gitseat.TagAnswer{Performed: true}) - } - return json.Marshal(meshcatalog.VersionsAnswer{Module: "m"}) - } - found := &memFindings{vals: map[string][]byte{}} - rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, call, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} }, - &findings{module: builds.BuilderModule, w: found, logf: t.Logf, now: time.Now}, nil, t.Logf) - rv.filled.Store(true) - type oldItem struct { - key string - raw []byte - } - var olds []oldItem - for _, m := range []string{"m1", "m2", "m3"} { - v, err := rv.Reserve(ctx, m, "0.1", []string{m + ":a"}, versions.Source{Fingerprint: "src2:" + m}) + // The move of the old state, run twice, puts each item once (at another holder, so the pull of oci is not asked). + old, err := b.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: builds.StateBucket("mesh-delivery-builder", versions.PublishQueueState)}) if err != nil { t.Fatal(err) } - if err := rv.Built(ctx, m, v.Version, map[string]string{m + ":a": "sha256:a"}, nil); err != nil { - t.Fatal(err) + var tagsWritten []string + call := func(tool string, body any) (json.RawMessage, error) { + switch tool { + case "seat:" + gitseat.Seat + "." + gitseat.VerbTags: // m4's 0.2.0 is tagged at another commit (§796) + return json.Marshal(gitseat.TagsAnswer{Form: 2, Tags: []gitseat.Tag{{Tag: "modules/m4/v0.2.0", Commit: "c-other"}}}) + case "seat:" + gitseat.Seat + ".tag": + tagsWritten = append(tagsWritten, body.(gitseat.TagInput).Tag+"@"+body.(gitseat.TagInput).Commit) + return json.Marshal(gitseat.TagAnswer{Performed: true}) + } + return json.Marshal(meshcatalog.VersionsAnswer{Module: "m"}) } - raw, _ := json.Marshal(versions.QueuedPublish{V: 1, Holder: "go", Module: m, Version: v.Version, Artifacts: map[string]string{m + ":a": "sha256:a"}, Class: "asked-again-at-start"}) - if _, err := old.Put(ctx, versions.QueueKey("go", m, v.Version), raw); err != nil { - t.Fatal(err) + found := &memFindings{vals: map[string][]byte{}} + rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, call, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} }, + &findings{module: builds.BuilderModule, w: found, logf: t.Logf, now: time.Now}, nil, t.Logf) + rv.filled.Store(true) + type oldItem struct { + key string + raw []byte } - olds = append(olds, oldItem{versions.QueueKey("go", m, v.Version), raw}) - } - rp := &versions.Republisher{Reserved: rv.r, Put: func(ctx context.Context, q versions.QueuedPublish) (bool, error) { - data, _ := json.Marshal(q) - ack, err := b.js.Publish(ctx, pq.subject(bind, q), data, jetstream.WithMsgID(versions.PublishMsgID(q.Holder, q.Class, q.Module, q.Version))) + var olds []oldItem + for _, m := range []string{"m1", "m2", "m3"} { + v, err := rv.Reserve(ctx, m, "0.1", []string{m + ":a"}, versions.Source{Fingerprint: "src2:" + m}) + if err != nil { + t.Fatal(err) + } + if err := rv.Built(ctx, m, v.Version, map[string]string{m + ":a": "sha256:a"}, nil); err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(versions.QueuedPublish{V: 1, Holder: "go", Module: m, Version: v.Version, Artifacts: map[string]string{m + ":a": "sha256:a"}, Class: "asked-again-at-start"}) + if _, err := old.Put(ctx, versions.QueueKey("go", m, v.Version), raw); err != nil { + t.Fatal(err) + } + olds = append(olds, oldItem{versions.QueueKey("go", m, v.Version), raw}) + } + rp := &versions.Republisher{Reserved: rv.r, Put: func(ctx context.Context, q versions.QueuedPublish) (bool, error) { + data, _ := json.Marshal(q) + ack, err := b.js.Publish(ctx, pq.subject(bind, q), data, jetstream.WithMsgID(versions.PublishMsgID(q.Holder, q.Class, q.Module, q.Version))) + if err != nil { + return false, err + } + return ack.Duplicate, nil + }, Started: time.Now(), Logf: t.Logf} + oldBucket := builds.StateBucket("mesh-delivery-builder", versions.PublishQueueState) + moveOld(ctx, b.js, oldBucket, rp, t.Logf) + if _, err := b.js.KeyValue(ctx, oldBucket); !errors.Is(err, jetstream.ErrBucketNotFound) { + t.Errorf("after the move the old state's bucket answers %v, want it removed by the builder (H-A-461 (1))", err) + } + // A start that stopped before the removal moves the same items again: the duplicates window keeps each once. + again, err := b.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: oldBucket}) if err != nil { - return false, err - } - return ack.Duplicate, nil - }, Started: time.Now(), Logf: t.Logf} - oldBucket := builds.StateBucket("mesh-delivery-builder", versions.PublishQueueState) - moveOld(ctx, b.js, oldBucket, rp, t.Logf) - if _, err := b.js.KeyValue(ctx, oldBucket); !errors.Is(err, jetstream.ErrBucketNotFound) { - t.Errorf("after the move the old state's bucket answers %v, want it removed by the builder (H-A-461 (1))", err) - } - // A start that stopped before the removal moves the same items again: the duplicates window keeps each once. - again, err := b.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: oldBucket}) - if err != nil { - t.Fatal(err) - } - for _, e := range olds { - if _, err := again.Put(ctx, e.key, e.raw); err != nil { t.Fatal(err) } - } - moveOld(ctx, b.js, oldBucket, rp, t.Logf) - moveOld(ctx, b.js, oldBucket, rp, t.Logf) // no bucket left: nothing moves - if _, err := b.js.KeyValue(ctx, oldBucket); !errors.Is(err, jetstream.ErrBucketNotFound) { - t.Errorf("after the second move the old state's bucket answers %v, want it removed", err) - } - st, err := b.js.Stream(ctx, bind.Stream) - if err != nil { - t.Fatal(err) - } - info, err := st.Info(ctx, jetstream.WithSubjectFilter(bind.Subject+".publish.go.>")) - if err != nil { - t.Fatal(err) - } - if len(info.State.Subjects) != 3 || info.State.Subjects[bind.Subject+".publish.go.again.m1.0=2E1=2E0"] != 1 { - t.Errorf("the move run twice left %v on the work queue: want each of the three once, as again", info.State.Subjects) - } + for _, e := range olds { + if _, err := again.Put(ctx, e.key, e.raw); err != nil { + t.Fatal(err) + } + } + moveOld(ctx, b.js, oldBucket, rp, t.Logf) + moveOld(ctx, b.js, oldBucket, rp, t.Logf) // no bucket left: nothing moves + if _, err := b.js.KeyValue(ctx, oldBucket); !errors.Is(err, jetstream.ErrBucketNotFound) { + t.Errorf("after the second move the old state's bucket answers %v, want it removed", err) + } + st, err := b.js.Stream(ctx, bind.Stream) + if err != nil { + t.Fatal(err) + } + info, err := st.Info(ctx, jetstream.WithSubjectFilter(bind.Subject+".publish.go.>")) + if err != nil { + t.Fatal(err) + } + if len(info.State.Subjects) != 3 || info.State.Subjects[bind.Subject+".publish.go.again.m1.0=2E1=2E0"] != 1 { + t.Errorf("the move run twice left %v on the work queue: want each of the three once, as again", info.State.Subjects) + } - // Design 47 H2 §796 (whole, H-A-453 (3)): a module's tags are read before it reserves; a number whose tag stands at another - // commit is skipped, the next number taken and version-tagged-elsewhere raised; the release tags /v at - // the build's commit, and the module's version-released clears the finding. - rv.r.Names = func(context.Context, string, string) (string, bool, error) { - return "git.example/o/r/modules/m4", true, nil - } - // Design 47 H2 §821, §805: the reservation is said as the seat mesh-builds' version-reserved, said as the process says it - // (sayReservedOn), and waited on by the seat and the event's name. - rv.names, rv.sayReserved = rv.r.Names, sayReservedOn(b.js, process.Account{Module: builds.BuilderModule, Node: "n1"}) - said, err := nats.Connect(b.srvURL, nats.UserInfo("ctl", "pw")) - if err != nil { - t.Fatal(err) - } - defer said.Close() - reservedCh := make(chan *nats.Msg, 4) - rs, err := said.ChanSubscribe(meshbuilds.SubjectVersionReserved, reservedCh) - if err != nil { - t.Fatal(err) - } - defer rs.Unsubscribe() - if err := said.Flush(); err != nil { - t.Fatal(err) - } - v4, err := rv.Reserve(ctx, "m4", "0.2", []string{"m4:lib"}, versions.Source{Fingerprint: "src2:m4", GoPackage: true, GoArtifact: "lib", - Repository: "o/r", Commit: "c-mine", Branch: "main", CatalogRevision: 1}) - elsewhere := views.FindingKey(meshconditionsclient.ScopeModule, "m4", versions.VersionTaggedElsewhere) - if _, raised := found.vals[elsewhere]; err != nil || v4.Version != "0.2.1" || v4.Tag != "modules/m4/v0.2.1" || !raised { - t.Fatalf("m4 reserved %+v (%v), %s raised %v: want 0.2.1 past 0.2.0's tag at another commit, its tag modules/m4/v0.2.1, and the finding", v4, err, elsewhere, raised) - } - select { - case m := <-reservedCh: - ev, _, err := meshbuilds.DecodeVersionReservedEvent(m.Data) - if err != nil || ev.Module != "m4" || ev.Artifact != "m4:lib" || ev.Path != "git.example/o/r/modules/m4" || ev.Version != "0.2.1" || - ev.Fingerprint != "src2:m4" || ev.Revision != "main.1" || m.Header.Get("x-source") != builds.BuilderModule { - t.Errorf("version-reserved = %+v (%v) from %q, want m4:lib at 0.2.1, its path, fingerprint src2:m4 and revision main.1, said by the builder", ev, err, m.Header.Get("x-source")) + // Design 47 H2 §796 (whole, H-A-453 (3)): a module's tags are read before it reserves; a number whose tag stands at another + // commit is skipped, the next number taken and version-tagged-elsewhere raised; the release tags /v at + // the build's commit, and the module's version-released clears the finding. + rv.r.Names = func(context.Context, string, string) (string, bool, error) { + return "git.example/o/r/modules/m4", true, nil } - case <-time.After(10 * time.Second): - t.Fatal("no version-reserved of m4's reservation within 10 s on " + meshbuilds.SubjectVersionReserved) - } - catalogEvents, err := b.js.StreamNameBySubject(ctx, meshcatalog.SubjectVersionReleased) // the bed's stream of the catalog's events - if err != nil { - t.Fatal(err) - } - evs, err := b.js.CreateOrUpdateConsumer(ctx, catalogEvents, jetstream.ConsumerConfig{Durable: "builder-events", AckPolicy: jetstream.AckExplicitPolicy, - FilterSubject: meshcatalog.SubjectVersionReleased, DeliverPolicy: jetstream.DeliverNewPolicy}) - if err != nil { - t.Fatal(err) - } - next := func(module, version string) jetstream.Msg { - t.Helper() - data, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: module, Version: version}) - if _, err := b.js.Publish(ctx, meshcatalog.SubjectVersionReleased, data); err != nil { - t.Fatal(err) - } - m, err := evs.Next(jetstream.FetchMaxWait(10 * time.Second)) + // Design 47 H2 §821, §805: the reservation is said as the seat mesh-builds' version-reserved, said as the process says it + // (sayReservedOn), and waited on by the seat and the event's name. + rv.names, rv.sayReserved = rv.r.Names, sayReservedOn(b.js, process.Account{Module: builds.BuilderModule, Node: "n1"}) + said, err := nats.Connect(b.srvURL, nats.UserInfo("ctl", "pw")) if err != nil { t.Fatal(err) } - return m - } - if !rv.heard(ctx, next("m4", "0.2.1")) { - t.Fatal("the reservations did not take version-released") - } - if _, raised := found.vals[elsewhere]; raised || !slices.Equal(tagsWritten, []string{"modules/m4/v0.2.1@c-mine"}) { - t.Errorf("after version-released %s raised %v, tags written %v: want it cleared and modules/m4/v0.2.1 written at c-mine, no tag moved", elsewhere, raised, tagsWritten) - } + defer said.Close() + reservedCh := make(chan *nats.Msg, 4) + rs, err := said.ChanSubscribe(meshbuilds.SubjectVersionReserved, reservedCh) + if err != nil { + t.Fatal(err) + } + defer rs.Unsubscribe() + if err := said.Flush(); err != nil { + t.Fatal(err) + } + v4, err := rv.Reserve(ctx, "m4", "0.2", []string{"m4:lib"}, versions.Source{Fingerprint: "src2:m4", GoPackage: true, GoArtifact: "lib", + Repository: "o/r", Commit: "c-mine", Branch: "main", CatalogRevision: 1}) + elsewhere := views.FindingKey(meshconditionsclient.ScopeModule, "m4", versions.VersionTaggedElsewhere) + if _, raised := found.vals[elsewhere]; err != nil || v4.Version != "0.2.1" || v4.Tag != "modules/m4/v0.2.1" || !raised { + t.Fatalf("m4 reserved %+v (%v), %s raised %v: want 0.2.1 past 0.2.0's tag at another commit, its tag modules/m4/v0.2.1, and the finding", v4, err, elsewhere, raised) + } + select { + case m := <-reservedCh: + ev, _, err := meshbuilds.DecodeVersionReservedEvent(m.Data) + if err != nil || ev.Module != "m4" || ev.Artifact != "m4:lib" || ev.Path != "git.example/o/r/modules/m4" || ev.Version != "0.2.1" || + ev.Fingerprint != "src2:m4" || ev.Revision != "main.1" || m.Header.Get("x-source") != builds.BuilderModule { + t.Errorf("version-reserved = %+v (%v) from %q, want m4:lib at 0.2.1, its path, fingerprint src2:m4 and revision main.1, said by the builder", ev, err, m.Header.Get("x-source")) + } + case <-time.After(10 * time.Second): + t.Fatal("no version-reserved of m4's reservation within 10 s on " + meshbuilds.SubjectVersionReserved) + } + catalogEvents, err := b.js.StreamNameBySubject(ctx, meshcatalog.SubjectVersionReleased) // the bed's stream of the catalog's events + if err != nil { + t.Fatal(err) + } + evs, err := b.js.CreateOrUpdateConsumer(ctx, catalogEvents, jetstream.ConsumerConfig{Durable: "builder-events", AckPolicy: jetstream.AckExplicitPolicy, + FilterSubject: meshcatalog.SubjectVersionReleased, DeliverPolicy: jetstream.DeliverNewPolicy}) + if err != nil { + t.Fatal(err) + } + next := func(module, version string) jetstream.Msg { + t.Helper() + data, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: module, Version: version}) + if _, err := b.js.Publish(ctx, meshcatalog.SubjectVersionReleased, data); err != nil { + t.Fatal(err) + } + m, err := evs.Next(jetstream.FetchMaxWait(10 * time.Second)) + if err != nil { + t.Fatal(err) + } + return m + } + if !rv.heard(ctx, next("m4", "0.2.1")) { + t.Fatal("the reservations did not take version-released") + } + if _, raised := found.vals[elsewhere]; raised || !slices.Equal(tagsWritten, []string{"modules/m4/v0.2.1@c-mine"}) { + t.Errorf("after version-released %s raised %v, tags written %v: want it cleared and modules/m4/v0.2.1 written at c-mine, no tag moved", elsewhere, raised, tagsWritten) + } - // Design 47 H2 §794 (whole, H-A-453 (3), H-A-462): a message the builder's handler leaves unacknowledged is kept on its - // consumer's ledger in the state consumers on the bus; held six minutes, the health names that backlog, and within five - // minutes it does not. - consumers, err := b.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: builds.StateBucket(builds.BuilderModule, process.ConsumersState)}) - if err != nil { - t.Fatal(err) - } - held := time.Now() - hub := &oneHandler{} - aged := agedHub{hub: hub, ledgers: &ledgers{logf: t.Logf, now: func() time.Time { return held }, - sink: consumersSink(process.AwaitBucket(ctx, b.js, nil, builds.StateBucket(builds.BuilderModule, process.ConsumersState), "", t.Logf))}} - _ = aged.Handle(meshcatalog.SubjectVersionReleased, func(m jetstream.Msg) { _ = m.Nak() }) // a handler standing still: asked again - hub.handle(next("m5", "0.1.0")) - state := versions.Reader{Keys: func() ([]string, error) { - l, err := consumers.ListKeys(ctx) + // Design 47 H2 §794 (whole, H-A-453 (3), H-A-462): a message the builder's handler leaves unacknowledged is kept on its + // consumer's ledger in the state consumers on the bus; held six minutes, the health names that backlog, and within five + // minutes it does not. + consumers, err := b.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: builds.StateBucket(builds.BuilderModule, process.ConsumersState)}) if err != nil { - return nil, err + t.Fatal(err) } - var keys []string - for k := range l.Keys() { - keys = append(keys, k) + held := time.Now() + hub := &oneHandler{} + aged := agedHub{hub: hub, ledgers: &ledgers{logf: t.Logf, now: func() time.Time { return held }, + sink: consumersSink(process.AwaitBucket(ctx, b.js, nil, builds.StateBucket(builds.BuilderModule, process.ConsumersState), "", t.Logf))}} + _ = aged.Handle(meshcatalog.SubjectVersionReleased, func(m jetstream.Msg) { _ = m.Nak() }) // a handler standing still: asked again + hub.handle(next("m5", "0.1.0")) + state := versions.Reader{Keys: func() ([]string, error) { + l, err := consumers.ListKeys(ctx) + if err != nil { + return nil, err + } + var keys []string + for k := range l.Keys() { + keys = append(keys, k) + } + return keys, nil + }, Get: func(k string) (json.RawMessage, bool, error) { + e, err := consumers.Get(ctx, k) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, false, nil + } + if err != nil { + return nil, false, err + } + return e.Value(), true, nil + }} + if a := versions.BuilderHealth(state, held.Add(6*time.Minute), 5*time.Minute); a.Healthy || !strings.Contains(a.Why, "builder-events (its oldest unacknowledged message waits 6m0s") { + t.Errorf("the health with a backlog held six minutes = %+v, want unhealthy naming builder-events", a) } - return keys, nil - }, Get: func(k string) (json.RawMessage, bool, error) { - e, err := consumers.Get(ctx, k) - if errors.Is(err, jetstream.ErrKeyNotFound) { - return nil, false, nil + if a := versions.BuilderHealth(state, held.Add(4*time.Minute), 5*time.Minute); !a.Healthy { + t.Errorf("the health four minutes in = %+v, want healthy", a) } - if err != nil { - return nil, false, err - } - return e.Value(), true, nil - }} - if a := versions.BuilderHealth(state, held.Add(6*time.Minute), 5*time.Minute); a.Healthy || !strings.Contains(a.Why, "builder-events (its oldest unacknowledged message waits 6m0s") { - t.Errorf("the health with a backlog held six minutes = %+v, want unhealthy naming builder-events", a) - } - if a := versions.BuilderHealth(state, held.Add(4*time.Minute), 5*time.Minute); !a.Healthy { - t.Errorf("the health four minutes in = %+v, want healthy", a) - } + }) } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go index ff2df922c..d2ad1ff18 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go @@ -659,12 +659,12 @@ func (pq *publishQueue) pull(ctx context.Context, holder string) error { } // dispatch runs the items of one holder, at most four at once, every live item before an again one (design 47 H2 §793, -// H-A-473 and its third addendum): on every turn no live item is held it pulls the live consumer itself, a fetch of up to its -// free slots that does not wait, and pulls the again consumer, or reads the live gap, only after that fetch returned nothing. When neither holds an item, a pull of one item waits on each, -// the bus's only wake for an item put or delivered again. An again item taken while the live pull waits is run only once the -// live consumer's delivered sequence is no further than the live items the dispatch received: a gap is a live item in flight -// to the waiting pull, and the again items are held until it arrives and runs. A pull the bus refuses ends the dispatch, said; -// the holder is pulled again when an item of it is put. +// H-A-473 and its third addendum): on every turn no live item is held it fetches live without waiting, up to its free slots, +// and takes again items only when that fetch returned nothing. With again items held it fetches live once more; only when +// that returns nothing are they run, and while the live pull waits only once the live consumer's delivered sequence is no +// further than the live items received: a gap is a live item in flight to the waiting pull, and the again items wait until it +// arrives and runs. When neither holds an item, a pull of one item waits on each, the bus's only wake for an item put or +// delivered again. A pull the bus refuses ends the dispatch, said; the holder is pulled again when an item of it is put. func (pq *publishQueue) dispatch(ctx context.Context, holder string, live, again jetstream.Consumer) { sem := make(chan struct{}, versions.PublishSendsAtOnce) run := func(m jetstream.Msg) { @@ -806,9 +806,20 @@ func (pq *publishQueue) dispatch(ctx context.Context, holder string, live, again } heldAgain = ms } - if len(heldAgain) > 0 && liveWaiting && inFlight() { + if len(heldAgain) > 0 { // with the again items held, live once more: one put since the first fetch goes first + ms, err := now(live, n) + if refused(err) { + failed(err) + return + } + gotLive(ms...) + got = ms + } + switch { + case len(got) > 0 || len(heldAgain) == 0: + case liveWaiting && inFlight(): held = true - } else if len(heldAgain) > 0 { + default: k := min(n, len(heldAgain)) got, heldAgain = heldAgain[:k], heldAgain[k:] }