From b914974b1d2ebeb8a2b6446ba54219aa0f82a57a Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 13:57:22 +0100 Subject: [PATCH 1/6] Judge test: the unknown top-level word is one no SDK knows The row proving that the judgement's strict read refuses an unknown top-level word used class, which the H2 SDK knows, so it failed at main. It now uses a word no SDK knows and proves the same refusal. Builds: design 47 H2 H-A-465, H-A-469 --- modules/mesh-delivery-indexer/judge/judge_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/mesh-delivery-indexer/judge/judge_test.go b/modules/mesh-delivery-indexer/judge/judge_test.go index ab91afb90..73649c782 100644 --- a/modules/mesh-delivery-indexer/judge/judge_test.go +++ b/modules/mesh-delivery-indexer/judge/judge_test.go @@ -27,9 +27,10 @@ func TestTheJudge(t *testing.T) { want: map[string]string{"a": `secret-`}}, {name: "an unknown field", raws: map[string]string{"a": `{"module":"a","version":"1.0","a-field-from-later":1}`}, want: map[string]string{"a": `"a-field-from-later"`}}, - // H-A-465: the take-in sets a top-level word it does not know aside (ForInputs); the judgement still refuses it. - {name: "a word a newer SDK adds", raws: map[string]string{"a": `{"module":"a","version":"1.0","class":"core"}`}, - want: map[string]string{"a": `"class"`}}, + // H-A-465: the take-in sets a top-level word it does not know aside (ForInputs); the judgement still refuses it. The + // word is one no SDK knows: class, the word of H-A-465's night, is known since H2. + {name: "a word a newer SDK adds", raws: map[string]string{"a": `{"module":"a","version":"1.0","zzz-unknown-word":"core"}`}, + want: map[string]string{"a": `"zzz-unknown-word"`}}, // Design 47 G §23, §28: a step's status is kept in the registration and never in a manifest. {name: "a manifest naming a step's status", raws: map[string]string{"a": `{"module":"a","version":"1.0","policy":{"lint":"bypass"}}`}, want: map[string]string{"a": `"policy"`}}, -- 2.54.0 From 1f33a4c04fc69329d75eda1b348044a05b44a743 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 14:14:15 +0100 Subject: [PATCH 2/6] Builder: the publish dispatch pulls the live consumer itself, then the again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dispatch read the live consumer's pending count against items the pull's callback had not yet handed it, so at one processor an again item was sent before a live one already delivered. It now fetches the live consumer without waiting and pulls the again consumer only when that returns nothing; an item is received when its fetch returns it. Builds: design 47 H2 §793; H-A-473 --- .../cmd/mesh-delivery-builder/publishitem.go | 178 +++++++++++++----- 1 file changed, 131 insertions(+), 47 deletions(-) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go index c31c62c8d..71cf6a75b 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go @@ -643,38 +643,28 @@ func (pq *publishQueue) pull(ctx context.Context, holder string) error { if err != nil { return err } - chans := map[string]chan jetstream.Msg{} - var live jetstream.Consumer - for _, class := range []string{versions.PublishClassStart, versions.PublishClassLive} { // the again first: the live is first by the dispatch, not by luck + cons := map[string]jetstream.Consumer{} + for _, class := range []string{versions.PublishClassStart, versions.PublishClassLive} { c, err := pq.js.CreateOrUpdateConsumer(ctx, b.Stream, jetstream.ConsumerConfig{Durable: publishesConsumer(holder, class), FilterSubject: b.Subject + ".publish." + versions.EscapeToken(holder) + "." + class + ".>", AckPolicy: jetstream.AckExplicitPolicy, AckWait: pq.ackWait, MaxAckPending: publishesMaxAckPending, MaxDeliver: publishesMaxDeliver}) if err != nil { return fmt.Errorf("the consumer %s: %w", publishesConsumer(holder, class), err) } - ch := make(chan jetstream.Msg, publishesMaxAckPending) - cc, err := c.Consume(func(m jetstream.Msg) { ch <- m }, jetstream.PullMaxMessages(publishesMaxAckPending)) - if err != nil { - return fmt.Errorf("the consumer %s is not pulled: %w", publishesConsumer(holder, class), err) - } - go func() { <-ctx.Done(); cc.Stop() }() - chans[class] = ch - if class == versions.PublishClassLive { - live = c - } - } - liveWaits := func() bool { - info, err := live.Info(ctx) - return err == nil && info.NumPending > 0 + cons[class] = c } pq.logf("pulling the publishes of %s, the live first", holder) - go pq.dispatch(ctx, chans[versions.PublishClassLive], chans[versions.PublishClassStart], liveWaits) + go pq.dispatch(ctx, holder, cons[versions.PublishClassLive], cons[versions.PublishClassStart]) return nil } -// dispatch runs the items of one holder, at most four at once, every live item before an again one: an again item pulled while -// a live one waits on the queue (liveWaits) runs after it. -func (pq *publishQueue) dispatch(ctx context.Context, live, again <-chan jetstream.Msg, liveWaits func() bool) { +// dispatch runs the items of one holder, at most four at once, every live item before an again one (design 47 H2 §793, +// H-A-473): it pulls the live consumer itself, a fetch of up to its free slots that does not wait, and pulls the again consumer +// only when that fetch returns nothing; an item is received the moment its fetch returns it, so no live item is delivered and +// not yet received while an again one is chosen. When neither holds an item, a pull of one item waits on each, and what it +// receives is run, the live first. A pull the bus refuses ends the dispatch, said; the holder is pulled again when an item of +// it is put. +func (pq *publishQueue) dispatch(ctx context.Context, holder string, live, again jetstream.Consumer) { sem := make(chan struct{}, versions.PublishSendsAtOnce) run := func(m jetstream.Msg) { go func() { @@ -682,48 +672,142 @@ func (pq *publishQueue) dispatch(ctx context.Context, live, again <-chan jetstre pq.take(ctx, m) }() } - var spare jetstream.Msg // an again item pulled while a live one waited + failed := func(err error) { + pq.logf("the publishes of %s are not pulled: %v; they are pulled when an item of it is put", holder, err) + pq.mu.Lock() + delete(pq.pulling, holder) + pq.mu.Unlock() + } + refused := func(err error) bool { + return err != nil && !errors.Is(err, jetstream.ErrNoMessages) && !errors.Is(err, nats.ErrTimeout) && ctx.Err() == nil + } + now := func(c jetstream.Consumer, n int) ([]jetstream.Msg, error) { + b, err := c.FetchNoWait(n) + if err != nil { + return nil, err + } + var got []jetstream.Msg + for m := range b.Messages() { + got = append(got, m) + } + return got, b.Error() + } + type waited struct { + m jetstream.Msg + err error + } + liveDone, againDone := make(chan waited, 1), make(chan waited, 1) + wait := func(c jetstream.Consumer, done chan<- waited) { + go func() { + var w waited + b, err := c.Fetch(1) + if err == nil { + for m := range b.Messages() { + w.m = m + } + err = b.Error() + } + w.err = err + done <- w + }() + } + var heldLive, heldAgain jetstream.Msg // an item a waiting pull received + var liveWaiting, againWaiting bool + heard := func(w waited, held *jetstream.Msg, waiting *bool) bool { + *waiting = false + if refused(w.err) { + failed(w.err) + return false + } + if w.m != nil { + *held = w.m + } + return true + } for { select { case sem <- struct{}{}: case <-ctx.Done(): return } - select { - case m := <-live: - run(m) - continue - default: - } - m := spare - spare = nil - if m == nil { + n := 1 + slots: + for n < cap(sem) { select { - case lm := <-live: - run(lm) - continue - case m = <-again: - case <-ctx.Done(): - return + case sem <- struct{}{}: + n++ + default: + break slots } } - if liveWaits() { - spare = m - select { - case lm := <-live: - run(lm) - case <-ctx.Done(): + select { + case w := <-liveDone: + if !heard(w, &heldLive, &liveWaiting) { return } - continue + default: } select { - case lm := <-live: // came while the queue was asked - spare = m - run(lm) + case w := <-againDone: + if !heard(w, &heldAgain, &againWaiting) { + return + } default: + } + var got []jetstream.Msg + switch { + case heldLive != nil: + got, heldLive = []jetstream.Msg{heldLive}, nil + case !liveWaiting: + ms, err := now(live, n) + if refused(err) { + failed(err) + return + } + got = ms + } + if len(got) == 0 { + switch { + case heldAgain != nil: + got, heldAgain = []jetstream.Msg{heldAgain}, nil + case !againWaiting: + ms, err := now(again, n) + if refused(err) { + failed(err) + return + } + got = ms + } + } + for _, m := range got { run(m) } + for range n - len(got) { + <-sem + } + if len(got) > 0 { + continue + } + if !liveWaiting { + liveWaiting = true + wait(live, liveDone) + } + if !againWaiting { + againWaiting = true + wait(again, againDone) + } + select { + case w := <-liveDone: + if !heard(w, &heldLive, &liveWaiting) { + return + } + case w := <-againDone: + if !heard(w, &heldAgain, &againWaiting) { + return + } + case <-ctx.Done(): + return + } } } -- 2.54.0 From 7f5f175b0580be0ebe83b0e98e09cdbf09853883 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 14:16:37 +0100 Subject: [PATCH 3/6] Builder whole test: done builds under the level's set-up bound; the hang at its own MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The test ran the builder at a 1 s step bound throughout, so a loaded build node could fail a build that must end done. Those builds now run under testbus.SetupBound; the hung step's part starts the builder again at 1 s, and the build again after it under the set-up bound. Builds: design 47 H2 §793; H-A-473 (addendum) --- .../cmd/mesh-delivery-builder/integration_test.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go index 839891b0e..ca19716f6 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go @@ -40,6 +40,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/seatdef" + "git.novox.be/novox/mesh-sdk/go/testbus" "git.novox.be/novox/mesh-sdk/go/views/viewstest" "mesh-delivery-builder/internal/build" @@ -578,7 +579,10 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7 first := "src2:" + strings.Repeat("1", 64) b.moduleFP(t, "netcheck", netcheck, "modules/netcheck", bothToolchains, commit, first, 1) digest := b.moduleFP(t, "netcheck", netcheck, "modules/netcheck", bothToolchains, commit, fp, 2) - b.start(t, "n1", map[string]string{"MESH_BUILDER_STEP_BOUND": "1s"}) + // H-A-473 (addendum): the builds that must end done run under the level bound's set-up share, never a bound a loaded + // node cannot meet; the hung step's part below starts the builder again at a short bound of its own. + doneBound := map[string]string{"MESH_BUILDER_STEP_BOUND": testbus.SetupBound(t).String()} + b.start(t, "n1", doneBound) v := b.await(t, "netcheck", isState(builds.BuildDone)) for _, a := range []string{"tools-go", "tools-typescript"} { es := reg.of(t, "netcheck/"+a) @@ -693,6 +697,8 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7 } // DLV-B10: a build module's bound running out fails the trunk build, which is the builder's finding build-failed; the // fingerprint built again clears it with its done build, numbering nothing of the builder's own (H-A-159). + b.stop() + b.start(t, "n1", map[string]string{"MESH_BUILDER_STEP_BOUND": "1s"}) // a step that never ends runs out of any bound release := make(chan struct{}) b.fake.hang.Store(&release) third := "src2:" + strings.Repeat("4", 64) @@ -713,6 +719,8 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7 reg.awaitFinding(t, failed, true) b.fake.hang.Store(nil) close(release) + b.stop() // the build again must end done + b.start(t, "n1", doneBound) b.moduleFP(t, "netcheck", netcheck, "modules/netcheck", bothToolchains, "89abcdef0123456789abcdef0123456789abcdef", third, 3) // the entry moves to a later commit, same fingerprint b.awaitFP(t, "netcheck", third, isState(builds.BuildDone)) reg.awaitFinding(t, failed, false) -- 2.54.0 From b77b70a66c5c0e6993afcf9330100aecb5f7015e Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 14:30:22 +0100 Subject: [PATCH 4/6] Builder: an again item waits for a live one in flight to the idle dispatch's waiting pull MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The idle dispatch keeps a waiting pull on each consumer, the bus's only wake for an item put or delivered again. A live item the waiting pull was given but the dispatch had not yet received let an again item go first (review finding on 1f33a4c0, publishitem.go:700, 761, 797). An again item taken while the live pull waits now runs only once the live consumer's delivered sequence is no further than the live items received; a gap holds it until the live item arrives and runs. Builds: design 47 H2 §793, H-A-473 (third addendum) --- .../publishes_integration_test.go | 52 ++++++++++++- .../cmd/mesh-delivery-builder/publishitem.go | 78 +++++++++++++------ 2 files changed, 107 insertions(+), 23 deletions(-) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go index 103aac070..32c6ac8a2 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go @@ -31,7 +31,8 @@ import ( // A live publish enqueued behind 100 again items while the builder was away is sent first; a dropped item is never sent, and // put again within the duplicates window is not stored; the move of the old state run twice puts each item once, and the // builder removes the old state's bucket itself (H-A-461 (1)). The seat mesh-builds' verb publish-queue reads the stream: counts -// per holder and class, the oldest and newest, the live item listed first (H-A-454). Then §796's tag at another commit, its +// per holder and class, the oldest and newest, the live item listed first (H-A-454). An idle dispatch sends a live item put +// before an again one first, 50 rounds (H-A-473, third addendum). Then §796's tag at another commit, its // reservation's version-reserved waited on by name (§821), and // §794's backlog held six minutes, as steps (H-A-453 (3)). func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) { @@ -135,6 +136,55 @@ func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) { } mu.Unlock() + // An idle dispatch (H-A-473, its third addendum): its two pulls wait; a live item put, then an again one, is sent live first, + // every time, 50 rounds, each a holder of its own. + classCh := make(chan string, 2) + idle := &publishQueue{js: b.js, ackWait: time.Minute, logf: func(string, ...any) {}, binding: publishesOf("mesh-delivery-builder"), + take: func(_ context.Context, m itemMsg) { + var q versions.QueuedPublish + _ = json.Unmarshal(m.Data(), &q) + _ = m.Ack() + classCh <- q.Class + }} + for r := range 50 { + holder := fmt.Sprintf("idle%d", r) + idle.learn(ctx, holder) + for _, class := range []string{versions.PublishClassLive, versions.PublishClassStart} { // both pulls stand waiting + c, err := b.js.Consumer(ctx, bind.Stream, publishesConsumer(holder, class)) + if err != nil { + t.Fatal(err) + } + for { + info, err := c.Info(ctx) + if err != nil { + t.Fatal(err) + } + if info.NumWaiting > 0 { + break + } + } + } + for _, class := range []string{versions.PublishClassLive, versions.PublishClassStart} { + q := versions.QueuedPublish{V: 1, Holder: holder, Module: "m", Version: "0.1.0", Class: class, Queued: time.Now()} + data, _ := json.Marshal(q) + if _, err := b.js.Publish(ctx, idle.subject(bind, q), data); err != nil { + t.Fatal(err) + } + } + var order []string + for range 2 { + select { + case c := <-classCh: + order = append(order, c) + case <-time.After(30 * time.Second): + t.Fatalf("round %d: sent %v of a live and an again item", r, order) + } + } + if order[0] != versions.PublishClassLive { + t.Errorf("round %d: an idle dispatch sent %v, want the live item put first sent first", r, order) + } + } + // The move of the old state, run twice, puts each item once (at another holder, so the pull of oci is not asked). old, err := b.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: builds.StateBucket("mesh-delivery-builder", versions.PublishQueueState)}) if err != nil { diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go index 71cf6a75b..56b134604 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go @@ -637,7 +637,7 @@ func (pq *publishQueue) learn(ctx context.Context, holder string) { } // pull makes holder's two consumers and dispatches their items, the live class first, at most four sends at once: an again item -// waits while a live one waits on the queue. +// is run only when no live one waits on the queue or is in flight to the dispatch (dispatch, H-A-473). func (pq *publishQueue) pull(ctx context.Context, holder string) error { b, err := pq.binding() if err != nil { @@ -659,11 +659,12 @@ func (pq *publishQueue) pull(ctx context.Context, holder string) error { } // dispatch runs the items of one holder, at most four at once, every live item before an again one (design 47 H2 §793, -// H-A-473): it pulls the live consumer itself, a fetch of up to its free slots that does not wait, and pulls the again consumer -// only when that fetch returns nothing; an item is received the moment its fetch returns it, so no live item is delivered and -// not yet received while an again one is chosen. When neither holds an item, a pull of one item waits on each, and what it -// receives is run, the live first. A pull the bus refuses ends the dispatch, said; the holder is pulled again when an item of -// it is put. +// H-A-473 and its third addendum): it pulls the live consumer itself, a fetch of up to its free slots that does not wait, and +// pulls the again consumer only when that fetch returns nothing. When neither holds an item, a pull of one item waits on each, +// the bus's only wake for an item put or delivered again. An again item taken while the live pull waits is run only once the +// live consumer's delivered sequence is no further than the live items the dispatch received: a gap is a live item in flight +// to the waiting pull, and the again items are held until it arrives and runs. A pull the bus refuses ends the dispatch, said; +// the holder is pulled again when an item of it is put. func (pq *publishQueue) dispatch(ctx context.Context, holder string, live, again jetstream.Consumer) { sem := make(chan struct{}, versions.PublishSendsAtOnce) run := func(m jetstream.Msg) { @@ -681,6 +682,32 @@ func (pq *publishQueue) dispatch(ctx context.Context, holder string, live, again refused := func(err error) bool { return err != nil && !errors.Is(err, jetstream.ErrNoMessages) && !errors.Is(err, nats.ErrTimeout) && ctx.Err() == nil } + // received is the live consumer's highest delivery sequence the dispatch holds; it starts at what the consumer delivered + // before this dispatch, to pulls no longer there. + var received uint64 + if info, err := live.Info(ctx); err != nil { + if refused(err) { + failed(err) + return + } + } else { + received = info.Delivered.Consumer + } + gotLive := func(ms ...jetstream.Msg) { + for _, m := range ms { + if md, err := m.Metadata(); err == nil && md.Sequence.Consumer > received { + received = md.Sequence.Consumer + } + } + } + inFlight := func() bool { // a live item delivered to the waiting pull and not yet received + info, err := live.Info(ctx) + if err != nil { + pq.logf("the publishes of %s: the live consumer cannot be read (%v); the again items wait for the live pull's answer", holder, err) + return true + } + return info.Delivered.Consumer > received + } now := func(c jetstream.Consumer, n int) ([]jetstream.Msg, error) { b, err := c.FetchNoWait(n) if err != nil { @@ -711,19 +738,22 @@ func (pq *publishQueue) dispatch(ctx context.Context, holder string, live, again done <- w }() } - var heldLive, heldAgain jetstream.Msg // an item a waiting pull received + var heldLive jetstream.Msg // a live item the waiting pull received + var heldAgain []jetstream.Msg // again items received, not yet run var liveWaiting, againWaiting bool - heard := func(w waited, held *jetstream.Msg, waiting *bool) bool { + heard := func(w waited, waiting *bool, keep func(jetstream.Msg)) bool { *waiting = false if refused(w.err) { failed(w.err) return false } if w.m != nil { - *held = w.m + keep(w.m) } return true } + keepLive := func(m jetstream.Msg) { gotLive(m); heldLive = m } + keepAgain := func(m jetstream.Msg) { heldAgain = append(heldAgain, m) } for { select { case sem <- struct{}{}: @@ -742,41 +772,45 @@ func (pq *publishQueue) dispatch(ctx context.Context, holder string, live, again } select { case w := <-liveDone: - if !heard(w, &heldLive, &liveWaiting) { + if !heard(w, &liveWaiting, keepLive) { return } default: } select { case w := <-againDone: - if !heard(w, &heldAgain, &againWaiting) { + if !heard(w, &againWaiting, keepAgain) { return } default: } var got []jetstream.Msg - switch { - case heldLive != nil: + held := false // again items held for a live item in flight + if heldLive != nil { got, heldLive = []jetstream.Msg{heldLive}, nil - case !liveWaiting: + } else if !liveWaiting || len(heldAgain) > 0 { ms, err := now(live, n) if refused(err) { failed(err) return } + gotLive(ms...) got = ms } if len(got) == 0 { - switch { - case heldAgain != nil: - got, heldAgain = []jetstream.Msg{heldAgain}, nil - case !againWaiting: + if len(heldAgain) == 0 && !againWaiting { ms, err := now(again, n) if refused(err) { failed(err) return } - got = ms + heldAgain = ms + } + if len(heldAgain) > 0 && liveWaiting && inFlight() { + held = true + } else if len(heldAgain) > 0 { + k := min(n, len(heldAgain)) + got, heldAgain = heldAgain[:k], heldAgain[k:] } } for _, m := range got { @@ -792,17 +826,17 @@ func (pq *publishQueue) dispatch(ctx context.Context, holder string, live, again liveWaiting = true wait(live, liveDone) } - if !againWaiting { + if !againWaiting && !held { againWaiting = true wait(again, againDone) } select { case w := <-liveDone: - if !heard(w, &heldLive, &liveWaiting) { + if !heard(w, &liveWaiting, keepLive) { return } case w := <-againDone: - if !heard(w, &heldAgain, &againWaiting) { + if !heard(w, &againWaiting, keepAgain) { return } case <-ctx.Done(): -- 2.54.0 From c37a7317b70c3b0eb7be0e1521364484afa8449a Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 14:35:41 +0100 Subject: [PATCH 5/6] Builder: every again fetch and gap read follows a live fetch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On the turn after an again item ran, the dispatch fetched again and read the live gap without fetching live first, so a live item the stream held but had not yet given to the waiting live pull showed no gap and a later again item went first (review round 2 on b77b70a6, publishitem.go:791). The live consumer is now fetched without waiting on every turn no live item is held. The test row adds that case and bounds its rounds by the level's set-up share. Builds: design 47 H2 §793, H-A-473 (third addendum) --- .../publishes_integration_test.go | 42 ++++++++++++------- .../cmd/mesh-delivery-builder/publishitem.go | 6 +-- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go index 32c6ac8a2..e05709ad0 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go @@ -22,6 +22,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/testbus" "git.novox.be/novox/mesh-sdk/go/views" "mesh-delivery-builder/internal/versions" @@ -136,16 +137,18 @@ func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) { } mu.Unlock() - // An idle dispatch (H-A-473, its third addendum): its two pulls wait; a live item put, then an again one, is sent live first, - // every time, 50 rounds, each a holder of its own. - classCh := make(chan string, 2) + // An idle dispatch (H-A-473, its third addendum): its two pulls wait; a live item put, then an again one, is sent live first; + // and an again item put after a live one, while the dispatch runs an again item put before both, is sent after the live one. + // Every time, 50 rounds, each a holder of its own. + sentOf := make(chan string, 8) idle := &publishQueue{js: b.js, ackWait: time.Minute, logf: func(string, ...any) {}, binding: publishesOf("mesh-delivery-builder"), take: func(_ context.Context, m itemMsg) { var q versions.QueuedPublish _ = json.Unmarshal(m.Data(), &q) _ = m.Ack() - classCh <- q.Class + sentOf <- q.Module }} + roundBound := testbus.SetupBound(t) // the level's set-up share (MESH_TEST_LEVEL_BOUND), as the other whole tests for r := range 50 { holder := fmt.Sprintf("idle%d", r) idle.learn(ctx, holder) @@ -164,25 +167,36 @@ func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) { } } } - for _, class := range []string{versions.PublishClassLive, versions.PublishClassStart} { - q := versions.QueuedPublish{V: 1, Holder: holder, Module: "m", Version: "0.1.0", Class: class, Queued: time.Now()} + put := func(module, class string) { + q := versions.QueuedPublish{V: 1, Holder: holder, Module: module, Version: "0.1.0", Class: class, Queued: time.Now()} data, _ := json.Marshal(q) if _, err := b.js.Publish(ctx, idle.subject(bind, q), data); err != nil { t.Fatal(err) } } - var order []string - for range 2 { - select { - case c := <-classCh: - order = append(order, c) - case <-time.After(30 * time.Second): - t.Fatalf("round %d: sent %v of a live and an again item", r, order) + sent := func(n int) []string { + var order []string + for range n { + select { + case m := <-sentOf: + order = append(order, m) + case <-time.After(roundBound): + t.Fatalf("round %d: sent %v of %d items", r, order, n) + } } + return order } - if order[0] != versions.PublishClassLive { + put("live", versions.PublishClassLive) + put("again", versions.PublishClassStart) + if order := sent(2); order[0] != "live" { t.Errorf("round %d: an idle dispatch sent %v, want the live item put first sent first", r, order) } + put("before", versions.PublishClassStart) + put("live2", versions.PublishClassLive) + put("after", versions.PublishClassStart) + if order := sent(3); slices.Index(order, "after") < slices.Index(order, "live2") { + t.Errorf("round %d: the dispatch sent %v, want the again item put after the live one sent after it", r, order) + } } // The move of the old state, run twice, puts each item once (at another holder, so the pull of oci is not asked). diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go index 56b134604..ff2df922c 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go @@ -659,8 +659,8 @@ func (pq *publishQueue) pull(ctx context.Context, holder string) error { } // dispatch runs the items of one holder, at most four at once, every live item before an again one (design 47 H2 §793, -// H-A-473 and its third addendum): it pulls the live consumer itself, a fetch of up to its free slots that does not wait, and -// pulls the again consumer only when that fetch returns nothing. When neither holds an item, a pull of one item waits on each, +// H-A-473 and its third addendum): on every turn no live item is held it pulls the live consumer itself, a fetch of up to its +// free slots that does not wait, and pulls the again consumer, or reads the live gap, only after that fetch returned nothing. When neither holds an item, a pull of one item waits on each, // the bus's only wake for an item put or delivered again. An again item taken while the live pull waits is run only once the // live consumer's delivered sequence is no further than the live items the dispatch received: a gap is a live item in flight // to the waiting pull, and the again items are held until it arrives and runs. A pull the bus refuses ends the dispatch, said; @@ -788,7 +788,7 @@ func (pq *publishQueue) dispatch(ctx context.Context, holder string, live, again held := false // again items held for a live item in flight if heldLive != nil { got, heldLive = []jetstream.Msg{heldLive}, nil - } else if !liveWaiting || len(heldAgain) > 0 { + } else { ms, err := now(live, n) if refused(err) { failed(err) -- 2.54.0 From 7e103f3221206167d445a2b439d05fcadb7d0789 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 14:40:17 +0100 Subject: [PATCH 6/6] Builder: with again items held, fetch live once more before the gap read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A live item put after the turn's live fetch and before the again items were held was not yet delivered at the gap read, so no gap showed and the later again item went first (review round 3 on c37a7317, publishitem.go:792-809). With again items held the dispatch now fetches live without waiting once more: what it returns runs and the again items stay held; only an empty fetch leads to the gap read. The test's idle-dispatch case is a subtest of its own, so it runs alone. Builds: design 47 H2 §793, H-A-473 (third addendum) --- .../publishes_integration_test.go | 594 +++++++++--------- .../cmd/mesh-delivery-builder/publishitem.go | 27 +- 2 files changed, 318 insertions(+), 303 deletions(-) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go index e05709ad0..f10fd3886 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishes_integration_test.go @@ -45,98 +45,6 @@ func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) { Retention: jetstream.WorkQueuePolicy, Discard: jetstream.DiscardNew, Duplicates: 24 * time.Hour, MaxAge: 7 * 24 * time.Hour}); err != nil { t.Fatal(err) } - kv := &itemKV{vals: map[string][]byte{}, revs: map[string]uint64{}} - var mu sync.Mutex - var sent []string - published := map[string]bool{} // the modules whose publish is heard - sentCh := make(chan string, 200) - items := &publishItems{reasks: func() jetstream.KeyValue { return kv }, set: publishSettings{Pause: time.Second, PauseMax: time.Minute, Tries: 5}, - now: time.Now, logf: t.Logf, - reserved: func(_ context.Context, m, v string) (versions.ReservedVersion, bool, error) { - mu.Lock() - defer mu.Unlock() - return versions.ReservedVersion{Module: m, Version: v, Artifacts: map[string]string{m + ":a": "sha256:a"}, Published: map[string]bool{m + ":a": published[m]}, BuiltAt: time.Now()}, true, nil - }, - send: func(_ context.Context, q versions.QueuedPublish) versions.SendFate { - mu.Lock() - sent = append(sent, q.Module) - mu.Unlock() - sentCh <- q.Module - return versions.SendFate{Sent: true} - }} - pq := &publishQueue{js: b.js, ackWait: time.Minute, logf: t.Logf, take: items.take, binding: publishesOf("mesh-delivery-builder")} - item := func(module, class string) versions.QueuedPublish { - return versions.QueuedPublish{V: 1, Holder: "oci", Module: module, Version: "0.1.0", Artifacts: map[string]string{module + ":a": "sha256:a"}, Class: class, Queued: time.Now()} - } - away := func(q versions.QueuedPublish) { // put while the builder was away: no holder learnt, nothing pulled - data, _ := json.Marshal(q) - if _, err := b.js.Publish(ctx, pq.subject(bind, q), data, jetstream.WithMsgID(versions.PublishMsgID(q.Holder, q.Class, q.Module, q.Version))); err != nil { - t.Fatal(err) - } - } - for i := range 100 { - away(item(fmt.Sprintf("a%03d", i), versions.PublishClassStart)) - } - away(item("dropme", versions.PublishClassStart)) - away(item("live", versions.PublishClassLive)) - if answer, err := pq.drop(ctx, items, "dropme", "0.1.0", "op@node", "a test drops it"); err != nil { - t.Fatalf("drop: %v", err) - } else { - t.Log(answer) - } - if dup, err := pq.put(ctx, item("dropme", versions.PublishClassStart)); err != nil || !dup { - t.Errorf("a dropped item put again within the window: duplicate %v (%v), want it not stored", dup, err) - } - raw, err := publishQueueVerb(ctx, pq)("op@node", json.RawMessage(`{}`)) - if a, ok := raw.(meshbuilds.PublishQueueAnswer); err != nil || !ok || a.Total != 101 || len(a.Counts) != 2 || a.Counts[0].Class != versions.PublishClassStart || - a.Counts[0].Count != 100 || a.Counts[1].Count != 1 || a.Counts[0].Oldest.After(a.Counts[0].Newest) || len(a.Queue) != versions.PublishQueueListCap || - a.Queue[0].Module != "live" || a.Capped { - t.Errorf("publish-queue answered %+v (%v), want 101 items, oci's 100 again and 1 live, fifty listed, the live first", raw, err) - } - if err := pq.learnAll(ctx); err != nil { - t.Fatal(err) - } - select { - case first := <-sentCh: - if first != "live" { - t.Errorf("the first item sent is %s, want the live one, enqueued behind 100 again items", first) - } - case <-time.After(30 * time.Second): - t.Fatal("nothing was sent") - } - for range 4 { // the again items in flight: at most four held unacknowledged, their publish unheard - select { - case <-sentCh: - case <-time.After(30 * time.Second): - t.Fatal("the again items were not sent") - } - } - deadline := time.After(60 * time.Second) - for { // each publish sent is heard: its item acknowledged, the next sent - mu.Lock() - n, done := len(sent), slices.Clone(sent) - for _, m := range done { - published[m] = true - } - mu.Unlock() - if n >= 101 { - break - } - for _, m := range done { - items.heard(ctx, m, "0.1.0") - } - select { - case <-sentCh: - case <-deadline: - t.Fatalf("sent %d of 101", n) - } - } - mu.Lock() - if slices.Contains(sent, "dropme") || len(sent) != 101 { - t.Errorf("sent %d items (dropme among them: %v), want the 101 and never the dropped one", len(sent), slices.Contains(sent, "dropme")) - } - mu.Unlock() - // An idle dispatch (H-A-473, its third addendum): its two pulls wait; a live item put, then an again one, is sent live first; // and an again item put after a live one, while the dispatch runs an again item put before both, is sent after the live one. // Every time, 50 rounds, each a holder of its own. @@ -148,240 +56,336 @@ func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) { _ = m.Ack() sentOf <- q.Module }} - roundBound := testbus.SetupBound(t) // the level's set-up share (MESH_TEST_LEVEL_BOUND), as the other whole tests - for r := range 50 { - holder := fmt.Sprintf("idle%d", r) - idle.learn(ctx, holder) - for _, class := range []string{versions.PublishClassLive, versions.PublishClassStart} { // both pulls stand waiting - c, err := b.js.Consumer(ctx, bind.Stream, publishesConsumer(holder, class)) - if err != nil { - t.Fatal(err) - } - for { - info, err := c.Info(ctx) + t.Run("idle dispatch", func(t *testing.T) { + roundBound := testbus.SetupBound(t) // the level's set-up share (MESH_TEST_LEVEL_BOUND), as the other whole tests + for r := range 50 { + holder := fmt.Sprintf("idle%d", r) + idle.learn(ctx, holder) + for _, class := range []string{versions.PublishClassLive, versions.PublishClassStart} { // both pulls stand waiting + c, err := b.js.Consumer(ctx, bind.Stream, publishesConsumer(holder, class)) if err != nil { t.Fatal(err) } - if info.NumWaiting > 0 { - break + for { + info, err := c.Info(ctx) + if err != nil { + t.Fatal(err) + } + if info.NumWaiting > 0 { + break + } } } + put := func(module, class string) { + q := versions.QueuedPublish{V: 1, Holder: holder, Module: module, Version: "0.1.0", Class: class, Queued: time.Now()} + data, _ := json.Marshal(q) + if _, err := b.js.Publish(ctx, idle.subject(bind, q), data); err != nil { + t.Fatal(err) + } + } + sent := func(n int) []string { + var order []string + for range n { + select { + case m := <-sentOf: + order = append(order, m) + case <-time.After(roundBound): + t.Fatalf("round %d: sent %v of %d items", r, order, n) + } + } + return order + } + put("live", versions.PublishClassLive) + put("again", versions.PublishClassStart) + if order := sent(2); order[0] != "live" { + t.Errorf("round %d: an idle dispatch sent %v, want the live item put first sent first", r, order) + } + put("before", versions.PublishClassStart) + put("live2", versions.PublishClassLive) + put("after", versions.PublishClassStart) + if order := sent(3); slices.Index(order, "after") < slices.Index(order, "live2") { + t.Errorf("round %d: the dispatch sent %v, want the again item put after the live one sent after it", r, order) + } } - put := func(module, class string) { - q := versions.QueuedPublish{V: 1, Holder: holder, Module: module, Version: "0.1.0", Class: class, Queued: time.Now()} + }) + + t.Run("the queue, the move, tags and backlog", func(t *testing.T) { + kv := &itemKV{vals: map[string][]byte{}, revs: map[string]uint64{}} + var mu sync.Mutex + var sent []string + published := map[string]bool{} // the modules whose publish is heard + sentCh := make(chan string, 200) + items := &publishItems{reasks: func() jetstream.KeyValue { return kv }, set: publishSettings{Pause: time.Second, PauseMax: time.Minute, Tries: 5}, + now: time.Now, logf: t.Logf, + reserved: func(_ context.Context, m, v string) (versions.ReservedVersion, bool, error) { + mu.Lock() + defer mu.Unlock() + return versions.ReservedVersion{Module: m, Version: v, Artifacts: map[string]string{m + ":a": "sha256:a"}, Published: map[string]bool{m + ":a": published[m]}, BuiltAt: time.Now()}, true, nil + }, + send: func(_ context.Context, q versions.QueuedPublish) versions.SendFate { + mu.Lock() + sent = append(sent, q.Module) + mu.Unlock() + sentCh <- q.Module + return versions.SendFate{Sent: true} + }} + pq := &publishQueue{js: b.js, ackWait: time.Minute, logf: t.Logf, take: items.take, binding: publishesOf("mesh-delivery-builder")} + item := func(module, class string) versions.QueuedPublish { + return versions.QueuedPublish{V: 1, Holder: "oci", Module: module, Version: "0.1.0", Artifacts: map[string]string{module + ":a": "sha256:a"}, Class: class, Queued: time.Now()} + } + away := func(q versions.QueuedPublish) { // put while the builder was away: no holder learnt, nothing pulled data, _ := json.Marshal(q) - if _, err := b.js.Publish(ctx, idle.subject(bind, q), data); err != nil { + if _, err := b.js.Publish(ctx, pq.subject(bind, q), data, jetstream.WithMsgID(versions.PublishMsgID(q.Holder, q.Class, q.Module, q.Version))); err != nil { t.Fatal(err) } } - sent := func(n int) []string { - var order []string - for range n { - select { - case m := <-sentOf: - order = append(order, m) - case <-time.After(roundBound): - t.Fatalf("round %d: sent %v of %d items", r, order, n) - } + for i := range 100 { + away(item(fmt.Sprintf("a%03d", i), versions.PublishClassStart)) + } + away(item("dropme", versions.PublishClassStart)) + away(item("live", versions.PublishClassLive)) + if answer, err := pq.drop(ctx, items, "dropme", "0.1.0", "op@node", "a test drops it"); err != nil { + t.Fatalf("drop: %v", err) + } else { + t.Log(answer) + } + if dup, err := pq.put(ctx, item("dropme", versions.PublishClassStart)); err != nil || !dup { + t.Errorf("a dropped item put again within the window: duplicate %v (%v), want it not stored", dup, err) + } + raw, err := publishQueueVerb(ctx, pq)("op@node", json.RawMessage(`{}`)) + if a, ok := raw.(meshbuilds.PublishQueueAnswer); err != nil || !ok || a.Total != 101 || len(a.Counts) != 2 || a.Counts[0].Class != versions.PublishClassStart || + a.Counts[0].Count != 100 || a.Counts[1].Count != 1 || a.Counts[0].Oldest.After(a.Counts[0].Newest) || len(a.Queue) != versions.PublishQueueListCap || + a.Queue[0].Module != "live" || a.Capped { + t.Errorf("publish-queue answered %+v (%v), want 101 items, oci's 100 again and 1 live, fifty listed, the live first", raw, err) + } + if err := pq.learnAll(ctx); err != nil { + t.Fatal(err) + } + select { + case first := <-sentCh: + if first != "live" { + t.Errorf("the first item sent is %s, want the live one, enqueued behind 100 again items", first) } - return order + case <-time.After(30 * time.Second): + t.Fatal("nothing was sent") } - put("live", versions.PublishClassLive) - put("again", versions.PublishClassStart) - if order := sent(2); order[0] != "live" { - t.Errorf("round %d: an idle dispatch sent %v, want the live item put first sent first", r, order) + for range 4 { // the again items in flight: at most four held unacknowledged, their publish unheard + select { + case <-sentCh: + case <-time.After(30 * time.Second): + t.Fatal("the again items were not sent") + } } - put("before", versions.PublishClassStart) - put("live2", versions.PublishClassLive) - put("after", versions.PublishClassStart) - if order := sent(3); slices.Index(order, "after") < slices.Index(order, "live2") { - t.Errorf("round %d: the dispatch sent %v, want the again item put after the live one sent after it", r, order) + deadline := time.After(60 * time.Second) + for { // each publish sent is heard: its item acknowledged, the next sent + mu.Lock() + n, done := len(sent), slices.Clone(sent) + for _, m := range done { + published[m] = true + } + mu.Unlock() + if n >= 101 { + break + } + for _, m := range done { + items.heard(ctx, m, "0.1.0") + } + select { + case <-sentCh: + case <-deadline: + t.Fatalf("sent %d of 101", n) + } } - } + mu.Lock() + if slices.Contains(sent, "dropme") || len(sent) != 101 { + t.Errorf("sent %d items (dropme among them: %v), want the 101 and never the dropped one", len(sent), slices.Contains(sent, "dropme")) + } + mu.Unlock() - // The move of the old state, run twice, puts each item once (at another holder, so the pull of oci is not asked). - old, err := b.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: builds.StateBucket("mesh-delivery-builder", versions.PublishQueueState)}) - if err != nil { - t.Fatal(err) - } - var tagsWritten []string - call := func(tool string, body any) (json.RawMessage, error) { - switch tool { - case "seat:" + gitseat.Seat + "." + gitseat.VerbTags: // m4's 0.2.0 is tagged at another commit (§796) - return json.Marshal(gitseat.TagsAnswer{Form: 2, Tags: []gitseat.Tag{{Tag: "modules/m4/v0.2.0", Commit: "c-other"}}}) - case "seat:" + gitseat.Seat + ".tag": - tagsWritten = append(tagsWritten, body.(gitseat.TagInput).Tag+"@"+body.(gitseat.TagInput).Commit) - return json.Marshal(gitseat.TagAnswer{Performed: true}) - } - return json.Marshal(meshcatalog.VersionsAnswer{Module: "m"}) - } - found := &memFindings{vals: map[string][]byte{}} - rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, call, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} }, - &findings{module: builds.BuilderModule, w: found, logf: t.Logf, now: time.Now}, nil, t.Logf) - rv.filled.Store(true) - type oldItem struct { - key string - raw []byte - } - var olds []oldItem - for _, m := range []string{"m1", "m2", "m3"} { - v, err := rv.Reserve(ctx, m, "0.1", []string{m + ":a"}, versions.Source{Fingerprint: "src2:" + m}) + // The move of the old state, run twice, puts each item once (at another holder, so the pull of oci is not asked). + old, err := b.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: builds.StateBucket("mesh-delivery-builder", versions.PublishQueueState)}) if err != nil { t.Fatal(err) } - if err := rv.Built(ctx, m, v.Version, map[string]string{m + ":a": "sha256:a"}, nil); err != nil { - t.Fatal(err) + var tagsWritten []string + call := func(tool string, body any) (json.RawMessage, error) { + switch tool { + case "seat:" + gitseat.Seat + "." + gitseat.VerbTags: // m4's 0.2.0 is tagged at another commit (§796) + return json.Marshal(gitseat.TagsAnswer{Form: 2, Tags: []gitseat.Tag{{Tag: "modules/m4/v0.2.0", Commit: "c-other"}}}) + case "seat:" + gitseat.Seat + ".tag": + tagsWritten = append(tagsWritten, body.(gitseat.TagInput).Tag+"@"+body.(gitseat.TagInput).Commit) + return json.Marshal(gitseat.TagAnswer{Performed: true}) + } + return json.Marshal(meshcatalog.VersionsAnswer{Module: "m"}) } - raw, _ := json.Marshal(versions.QueuedPublish{V: 1, Holder: "go", Module: m, Version: v.Version, Artifacts: map[string]string{m + ":a": "sha256:a"}, Class: "asked-again-at-start"}) - if _, err := old.Put(ctx, versions.QueueKey("go", m, v.Version), raw); err != nil { - t.Fatal(err) + found := &memFindings{vals: map[string][]byte{}} + rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, call, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} }, + &findings{module: builds.BuilderModule, w: found, logf: t.Logf, now: time.Now}, nil, t.Logf) + rv.filled.Store(true) + type oldItem struct { + key string + raw []byte } - olds = append(olds, oldItem{versions.QueueKey("go", m, v.Version), raw}) - } - rp := &versions.Republisher{Reserved: rv.r, Put: func(ctx context.Context, q versions.QueuedPublish) (bool, error) { - data, _ := json.Marshal(q) - ack, err := b.js.Publish(ctx, pq.subject(bind, q), data, jetstream.WithMsgID(versions.PublishMsgID(q.Holder, q.Class, q.Module, q.Version))) + var olds []oldItem + for _, m := range []string{"m1", "m2", "m3"} { + v, err := rv.Reserve(ctx, m, "0.1", []string{m + ":a"}, versions.Source{Fingerprint: "src2:" + m}) + if err != nil { + t.Fatal(err) + } + if err := rv.Built(ctx, m, v.Version, map[string]string{m + ":a": "sha256:a"}, nil); err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(versions.QueuedPublish{V: 1, Holder: "go", Module: m, Version: v.Version, Artifacts: map[string]string{m + ":a": "sha256:a"}, Class: "asked-again-at-start"}) + if _, err := old.Put(ctx, versions.QueueKey("go", m, v.Version), raw); err != nil { + t.Fatal(err) + } + olds = append(olds, oldItem{versions.QueueKey("go", m, v.Version), raw}) + } + rp := &versions.Republisher{Reserved: rv.r, Put: func(ctx context.Context, q versions.QueuedPublish) (bool, error) { + data, _ := json.Marshal(q) + ack, err := b.js.Publish(ctx, pq.subject(bind, q), data, jetstream.WithMsgID(versions.PublishMsgID(q.Holder, q.Class, q.Module, q.Version))) + if err != nil { + return false, err + } + return ack.Duplicate, nil + }, Started: time.Now(), Logf: t.Logf} + oldBucket := builds.StateBucket("mesh-delivery-builder", versions.PublishQueueState) + moveOld(ctx, b.js, oldBucket, rp, t.Logf) + if _, err := b.js.KeyValue(ctx, oldBucket); !errors.Is(err, jetstream.ErrBucketNotFound) { + t.Errorf("after the move the old state's bucket answers %v, want it removed by the builder (H-A-461 (1))", err) + } + // A start that stopped before the removal moves the same items again: the duplicates window keeps each once. + again, err := b.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: oldBucket}) if err != nil { - return false, err - } - return ack.Duplicate, nil - }, Started: time.Now(), Logf: t.Logf} - oldBucket := builds.StateBucket("mesh-delivery-builder", versions.PublishQueueState) - moveOld(ctx, b.js, oldBucket, rp, t.Logf) - if _, err := b.js.KeyValue(ctx, oldBucket); !errors.Is(err, jetstream.ErrBucketNotFound) { - t.Errorf("after the move the old state's bucket answers %v, want it removed by the builder (H-A-461 (1))", err) - } - // A start that stopped before the removal moves the same items again: the duplicates window keeps each once. - again, err := b.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: oldBucket}) - if err != nil { - t.Fatal(err) - } - for _, e := range olds { - if _, err := again.Put(ctx, e.key, e.raw); err != nil { t.Fatal(err) } - } - moveOld(ctx, b.js, oldBucket, rp, t.Logf) - moveOld(ctx, b.js, oldBucket, rp, t.Logf) // no bucket left: nothing moves - if _, err := b.js.KeyValue(ctx, oldBucket); !errors.Is(err, jetstream.ErrBucketNotFound) { - t.Errorf("after the second move the old state's bucket answers %v, want it removed", err) - } - st, err := b.js.Stream(ctx, bind.Stream) - if err != nil { - t.Fatal(err) - } - info, err := st.Info(ctx, jetstream.WithSubjectFilter(bind.Subject+".publish.go.>")) - if err != nil { - t.Fatal(err) - } - if len(info.State.Subjects) != 3 || info.State.Subjects[bind.Subject+".publish.go.again.m1.0=2E1=2E0"] != 1 { - t.Errorf("the move run twice left %v on the work queue: want each of the three once, as again", info.State.Subjects) - } + for _, e := range olds { + if _, err := again.Put(ctx, e.key, e.raw); err != nil { + t.Fatal(err) + } + } + moveOld(ctx, b.js, oldBucket, rp, t.Logf) + moveOld(ctx, b.js, oldBucket, rp, t.Logf) // no bucket left: nothing moves + if _, err := b.js.KeyValue(ctx, oldBucket); !errors.Is(err, jetstream.ErrBucketNotFound) { + t.Errorf("after the second move the old state's bucket answers %v, want it removed", err) + } + st, err := b.js.Stream(ctx, bind.Stream) + if err != nil { + t.Fatal(err) + } + info, err := st.Info(ctx, jetstream.WithSubjectFilter(bind.Subject+".publish.go.>")) + if err != nil { + t.Fatal(err) + } + if len(info.State.Subjects) != 3 || info.State.Subjects[bind.Subject+".publish.go.again.m1.0=2E1=2E0"] != 1 { + t.Errorf("the move run twice left %v on the work queue: want each of the three once, as again", info.State.Subjects) + } - // Design 47 H2 §796 (whole, H-A-453 (3)): a module's tags are read before it reserves; a number whose tag stands at another - // commit is skipped, the next number taken and version-tagged-elsewhere raised; the release tags /v at - // the build's commit, and the module's version-released clears the finding. - rv.r.Names = func(context.Context, string, string) (string, bool, error) { - return "git.example/o/r/modules/m4", true, nil - } - // Design 47 H2 §821, §805: the reservation is said as the seat mesh-builds' version-reserved, said as the process says it - // (sayReservedOn), and waited on by the seat and the event's name. - rv.names, rv.sayReserved = rv.r.Names, sayReservedOn(b.js, process.Account{Module: builds.BuilderModule, Node: "n1"}) - said, err := nats.Connect(b.srvURL, nats.UserInfo("ctl", "pw")) - if err != nil { - t.Fatal(err) - } - defer said.Close() - reservedCh := make(chan *nats.Msg, 4) - rs, err := said.ChanSubscribe(meshbuilds.SubjectVersionReserved, reservedCh) - if err != nil { - t.Fatal(err) - } - defer rs.Unsubscribe() - if err := said.Flush(); err != nil { - t.Fatal(err) - } - v4, err := rv.Reserve(ctx, "m4", "0.2", []string{"m4:lib"}, versions.Source{Fingerprint: "src2:m4", GoPackage: true, GoArtifact: "lib", - Repository: "o/r", Commit: "c-mine", Branch: "main", CatalogRevision: 1}) - elsewhere := views.FindingKey(meshconditionsclient.ScopeModule, "m4", versions.VersionTaggedElsewhere) - if _, raised := found.vals[elsewhere]; err != nil || v4.Version != "0.2.1" || v4.Tag != "modules/m4/v0.2.1" || !raised { - t.Fatalf("m4 reserved %+v (%v), %s raised %v: want 0.2.1 past 0.2.0's tag at another commit, its tag modules/m4/v0.2.1, and the finding", v4, err, elsewhere, raised) - } - select { - case m := <-reservedCh: - ev, _, err := meshbuilds.DecodeVersionReservedEvent(m.Data) - if err != nil || ev.Module != "m4" || ev.Artifact != "m4:lib" || ev.Path != "git.example/o/r/modules/m4" || ev.Version != "0.2.1" || - ev.Fingerprint != "src2:m4" || ev.Revision != "main.1" || m.Header.Get("x-source") != builds.BuilderModule { - t.Errorf("version-reserved = %+v (%v) from %q, want m4:lib at 0.2.1, its path, fingerprint src2:m4 and revision main.1, said by the builder", ev, err, m.Header.Get("x-source")) + // Design 47 H2 §796 (whole, H-A-453 (3)): a module's tags are read before it reserves; a number whose tag stands at another + // commit is skipped, the next number taken and version-tagged-elsewhere raised; the release tags /v at + // the build's commit, and the module's version-released clears the finding. + rv.r.Names = func(context.Context, string, string) (string, bool, error) { + return "git.example/o/r/modules/m4", true, nil } - case <-time.After(10 * time.Second): - t.Fatal("no version-reserved of m4's reservation within 10 s on " + meshbuilds.SubjectVersionReserved) - } - catalogEvents, err := b.js.StreamNameBySubject(ctx, meshcatalog.SubjectVersionReleased) // the bed's stream of the catalog's events - if err != nil { - t.Fatal(err) - } - evs, err := b.js.CreateOrUpdateConsumer(ctx, catalogEvents, jetstream.ConsumerConfig{Durable: "builder-events", AckPolicy: jetstream.AckExplicitPolicy, - FilterSubject: meshcatalog.SubjectVersionReleased, DeliverPolicy: jetstream.DeliverNewPolicy}) - if err != nil { - t.Fatal(err) - } - next := func(module, version string) jetstream.Msg { - t.Helper() - data, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: module, Version: version}) - if _, err := b.js.Publish(ctx, meshcatalog.SubjectVersionReleased, data); err != nil { - t.Fatal(err) - } - m, err := evs.Next(jetstream.FetchMaxWait(10 * time.Second)) + // Design 47 H2 §821, §805: the reservation is said as the seat mesh-builds' version-reserved, said as the process says it + // (sayReservedOn), and waited on by the seat and the event's name. + rv.names, rv.sayReserved = rv.r.Names, sayReservedOn(b.js, process.Account{Module: builds.BuilderModule, Node: "n1"}) + said, err := nats.Connect(b.srvURL, nats.UserInfo("ctl", "pw")) if err != nil { t.Fatal(err) } - return m - } - if !rv.heard(ctx, next("m4", "0.2.1")) { - t.Fatal("the reservations did not take version-released") - } - if _, raised := found.vals[elsewhere]; raised || !slices.Equal(tagsWritten, []string{"modules/m4/v0.2.1@c-mine"}) { - t.Errorf("after version-released %s raised %v, tags written %v: want it cleared and modules/m4/v0.2.1 written at c-mine, no tag moved", elsewhere, raised, tagsWritten) - } + defer said.Close() + reservedCh := make(chan *nats.Msg, 4) + rs, err := said.ChanSubscribe(meshbuilds.SubjectVersionReserved, reservedCh) + if err != nil { + t.Fatal(err) + } + defer rs.Unsubscribe() + if err := said.Flush(); err != nil { + t.Fatal(err) + } + v4, err := rv.Reserve(ctx, "m4", "0.2", []string{"m4:lib"}, versions.Source{Fingerprint: "src2:m4", GoPackage: true, GoArtifact: "lib", + Repository: "o/r", Commit: "c-mine", Branch: "main", CatalogRevision: 1}) + elsewhere := views.FindingKey(meshconditionsclient.ScopeModule, "m4", versions.VersionTaggedElsewhere) + if _, raised := found.vals[elsewhere]; err != nil || v4.Version != "0.2.1" || v4.Tag != "modules/m4/v0.2.1" || !raised { + t.Fatalf("m4 reserved %+v (%v), %s raised %v: want 0.2.1 past 0.2.0's tag at another commit, its tag modules/m4/v0.2.1, and the finding", v4, err, elsewhere, raised) + } + select { + case m := <-reservedCh: + ev, _, err := meshbuilds.DecodeVersionReservedEvent(m.Data) + if err != nil || ev.Module != "m4" || ev.Artifact != "m4:lib" || ev.Path != "git.example/o/r/modules/m4" || ev.Version != "0.2.1" || + ev.Fingerprint != "src2:m4" || ev.Revision != "main.1" || m.Header.Get("x-source") != builds.BuilderModule { + t.Errorf("version-reserved = %+v (%v) from %q, want m4:lib at 0.2.1, its path, fingerprint src2:m4 and revision main.1, said by the builder", ev, err, m.Header.Get("x-source")) + } + case <-time.After(10 * time.Second): + t.Fatal("no version-reserved of m4's reservation within 10 s on " + meshbuilds.SubjectVersionReserved) + } + catalogEvents, err := b.js.StreamNameBySubject(ctx, meshcatalog.SubjectVersionReleased) // the bed's stream of the catalog's events + if err != nil { + t.Fatal(err) + } + evs, err := b.js.CreateOrUpdateConsumer(ctx, catalogEvents, jetstream.ConsumerConfig{Durable: "builder-events", AckPolicy: jetstream.AckExplicitPolicy, + FilterSubject: meshcatalog.SubjectVersionReleased, DeliverPolicy: jetstream.DeliverNewPolicy}) + if err != nil { + t.Fatal(err) + } + next := func(module, version string) jetstream.Msg { + t.Helper() + data, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: module, Version: version}) + if _, err := b.js.Publish(ctx, meshcatalog.SubjectVersionReleased, data); err != nil { + t.Fatal(err) + } + m, err := evs.Next(jetstream.FetchMaxWait(10 * time.Second)) + if err != nil { + t.Fatal(err) + } + return m + } + if !rv.heard(ctx, next("m4", "0.2.1")) { + t.Fatal("the reservations did not take version-released") + } + if _, raised := found.vals[elsewhere]; raised || !slices.Equal(tagsWritten, []string{"modules/m4/v0.2.1@c-mine"}) { + t.Errorf("after version-released %s raised %v, tags written %v: want it cleared and modules/m4/v0.2.1 written at c-mine, no tag moved", elsewhere, raised, tagsWritten) + } - // Design 47 H2 §794 (whole, H-A-453 (3), H-A-462): a message the builder's handler leaves unacknowledged is kept on its - // consumer's ledger in the state consumers on the bus; held six minutes, the health names that backlog, and within five - // minutes it does not. - consumers, err := b.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: builds.StateBucket(builds.BuilderModule, process.ConsumersState)}) - if err != nil { - t.Fatal(err) - } - held := time.Now() - hub := &oneHandler{} - aged := agedHub{hub: hub, ledgers: &ledgers{logf: t.Logf, now: func() time.Time { return held }, - sink: consumersSink(process.AwaitBucket(ctx, b.js, nil, builds.StateBucket(builds.BuilderModule, process.ConsumersState), "", t.Logf))}} - _ = aged.Handle(meshcatalog.SubjectVersionReleased, func(m jetstream.Msg) { _ = m.Nak() }) // a handler standing still: asked again - hub.handle(next("m5", "0.1.0")) - state := versions.Reader{Keys: func() ([]string, error) { - l, err := consumers.ListKeys(ctx) + // Design 47 H2 §794 (whole, H-A-453 (3), H-A-462): a message the builder's handler leaves unacknowledged is kept on its + // consumer's ledger in the state consumers on the bus; held six minutes, the health names that backlog, and within five + // minutes it does not. + consumers, err := b.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: builds.StateBucket(builds.BuilderModule, process.ConsumersState)}) if err != nil { - return nil, err + t.Fatal(err) } - var keys []string - for k := range l.Keys() { - keys = append(keys, k) + held := time.Now() + hub := &oneHandler{} + aged := agedHub{hub: hub, ledgers: &ledgers{logf: t.Logf, now: func() time.Time { return held }, + sink: consumersSink(process.AwaitBucket(ctx, b.js, nil, builds.StateBucket(builds.BuilderModule, process.ConsumersState), "", t.Logf))}} + _ = aged.Handle(meshcatalog.SubjectVersionReleased, func(m jetstream.Msg) { _ = m.Nak() }) // a handler standing still: asked again + hub.handle(next("m5", "0.1.0")) + state := versions.Reader{Keys: func() ([]string, error) { + l, err := consumers.ListKeys(ctx) + if err != nil { + return nil, err + } + var keys []string + for k := range l.Keys() { + keys = append(keys, k) + } + return keys, nil + }, Get: func(k string) (json.RawMessage, bool, error) { + e, err := consumers.Get(ctx, k) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, false, nil + } + if err != nil { + return nil, false, err + } + return e.Value(), true, nil + }} + if a := versions.BuilderHealth(state, held.Add(6*time.Minute), 5*time.Minute); a.Healthy || !strings.Contains(a.Why, "builder-events (its oldest unacknowledged message waits 6m0s") { + t.Errorf("the health with a backlog held six minutes = %+v, want unhealthy naming builder-events", a) } - return keys, nil - }, Get: func(k string) (json.RawMessage, bool, error) { - e, err := consumers.Get(ctx, k) - if errors.Is(err, jetstream.ErrKeyNotFound) { - return nil, false, nil + if a := versions.BuilderHealth(state, held.Add(4*time.Minute), 5*time.Minute); !a.Healthy { + t.Errorf("the health four minutes in = %+v, want healthy", a) } - if err != nil { - return nil, false, err - } - return e.Value(), true, nil - }} - if a := versions.BuilderHealth(state, held.Add(6*time.Minute), 5*time.Minute); a.Healthy || !strings.Contains(a.Why, "builder-events (its oldest unacknowledged message waits 6m0s") { - t.Errorf("the health with a backlog held six minutes = %+v, want unhealthy naming builder-events", a) - } - if a := versions.BuilderHealth(state, held.Add(4*time.Minute), 5*time.Minute); !a.Healthy { - t.Errorf("the health four minutes in = %+v, want healthy", a) - } + }) } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go index ff2df922c..d2ad1ff18 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go @@ -659,12 +659,12 @@ func (pq *publishQueue) pull(ctx context.Context, holder string) error { } // dispatch runs the items of one holder, at most four at once, every live item before an again one (design 47 H2 §793, -// H-A-473 and its third addendum): on every turn no live item is held it pulls the live consumer itself, a fetch of up to its -// free slots that does not wait, and pulls the again consumer, or reads the live gap, only after that fetch returned nothing. When neither holds an item, a pull of one item waits on each, -// the bus's only wake for an item put or delivered again. An again item taken while the live pull waits is run only once the -// live consumer's delivered sequence is no further than the live items the dispatch received: a gap is a live item in flight -// to the waiting pull, and the again items are held until it arrives and runs. A pull the bus refuses ends the dispatch, said; -// the holder is pulled again when an item of it is put. +// H-A-473 and its third addendum): on every turn no live item is held it fetches live without waiting, up to its free slots, +// and takes again items only when that fetch returned nothing. With again items held it fetches live once more; only when +// that returns nothing are they run, and while the live pull waits only once the live consumer's delivered sequence is no +// further than the live items received: a gap is a live item in flight to the waiting pull, and the again items wait until it +// arrives and runs. When neither holds an item, a pull of one item waits on each, the bus's only wake for an item put or +// delivered again. A pull the bus refuses ends the dispatch, said; the holder is pulled again when an item of it is put. func (pq *publishQueue) dispatch(ctx context.Context, holder string, live, again jetstream.Consumer) { sem := make(chan struct{}, versions.PublishSendsAtOnce) run := func(m jetstream.Msg) { @@ -806,9 +806,20 @@ func (pq *publishQueue) dispatch(ctx context.Context, holder string, live, again } heldAgain = ms } - if len(heldAgain) > 0 && liveWaiting && inFlight() { + if len(heldAgain) > 0 { // with the again items held, live once more: one put since the first fetch goes first + ms, err := now(live, n) + if refused(err) { + failed(err) + return + } + gotLive(ms...) + got = ms + } + switch { + case len(got) > 0 || len(heldAgain) == 0: + case liveWaiting && inFlight(): held = true - } else if len(heldAgain) > 0 { + default: k := min(n, len(heldAgain)) got, heldAgain = heldAgain[:k], heldAgain[k:] } -- 2.54.0