From 0f65240139f61fc499e8e4540632e9a317142dd8 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 11:38:43 +0100 Subject: [PATCH] builder: bind the seat's worker again after a deletion The builder bound the seat mesh-builds' worker once, under a sync.Once, and stopped hearing the bus's advisories after that, so a worker deleted and made again by the grants part was never pulled again and the asks piled up. The advisories CREATED and DELETED are now heard for as long as the builder runs: a deletion lets the worker go and says so, and a creation binds it again. The step test deletes the worker, makes it again and has an ask ended through it. Builds: design 47 H2 H-A-468 (2) --- .../step_integration_test.go | 25 ++++++ .../cmd/mesh-delivery-builder/stepqueue.go | 81 ++++++++++++------- 2 files changed, 78 insertions(+), 28 deletions(-) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/step_integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/step_integration_test.go index 982b2cbc7..500578b53 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/step_integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/step_integration_test.go @@ -113,4 +113,29 @@ func TestTheRunnersBuildStepAskIsAnsweredAsStepEnded(t *testing.T) { } runtime.Gosched() } + // H-A-468 (2): the worker deleted and made again by the broker's part is bound again, and an ask put after it is ended. + if err := b.js.DeleteConsumer(b.ctx, stream, worker); err != nil { + t.Fatal(err) + } + b.awaitSaid(t, "work queue "+worker+" on "+stream+" was deleted") + b.seatQueue(t) + b.awaitSaid(t, "pulling the seat "+meshbuilds.Seat+"'s asks from "+worker) + run.Pipeline = "main.3.netcheck" + data, _ = json.Marshal(run) + msg = nats.NewMsg(meshbuilds.AcceptSubjectStep) + msg.Data = data + msg.Header.Set("x-event-id", "ask-3") + msg.Header.Set(nats.MsgIdHdr, "ask-3") + if _, err := b.js.PublishMsg(b.ctx, msg); err != nil { + t.Fatal(err) + } + select { + case m := <-ended: + var end meshbuilds.StepEnded + if err := json.Unmarshal(m.Data, &end); err != nil || end.ID != "ask-3" || end.Outcome != meshbuilds.StepEndedOutcomePassed { + t.Fatalf("step-ended = %+v, %v; want the ask ask-3 passed through the worker made again", end, err) + } + case <-time.After(20 * time.Second): + t.Fatal("no step-ended for ask-3 within 20 s of the worker made again: the builder did not bind it again") + } } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/stepqueue.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/stepqueue.go index 29597fdbf..f664d419f 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/stepqueue.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/stepqueue.go @@ -255,46 +255,71 @@ func inProgressOf(def func() (seatdef.Definition, bool), ackWait func() time.Dur // serveSteps pulls the seat's work queue through its one shared worker, every ask waiting at once (at most the worker's // max ack pending), once the broker made it: a worker not there yet is said, and bound when the bus says it was created. +// The advisories are heard for as long as the builder runs (design 47 H2 H-A-468 (2)): a worker deleted is let go and said, +// and bound again when the bus says it was created again. func serveSteps(ctx context.Context, conn *nats.Conn, js jetstream.JetStream, q *stepQueue, ackWait *atomic.Int64) error { stream, worker := buscompose.SeatQueueStreamName(meshbuilds.Seat), buscompose.SeatQueueWorkerName(meshbuilds.Seat) - var once sync.Once + var mu sync.Mutex + var pulling jetstream.ConsumeContext bind := func() bool { + mu.Lock() + defer mu.Unlock() + if pulling != nil { + return true + } c, err := js.Consumer(ctx, stream, worker) if err != nil { return false } - bound := false - once.Do(func() { - bound = true - info := c.CachedInfo() - if info != nil && info.Config.AckWait > 0 { - ackWait.Store(int64(info.Config.AckWait)) - } - pending := 1000 - if info != nil && info.Config.MaxAckPending > 0 { - pending = info.Config.MaxAckPending - } - cc, err := c.Consume(func(m jetstream.Msg) { q.take(ctx, m) }, jetstream.PullMaxMessages(pending)) - if err != nil { - q.logf("the seat %s's work queue %s is not pulled: %v", meshbuilds.Seat, worker, err) - return - } - q.logf("pulling the seat %s's asks from %s", meshbuilds.Seat, worker) - go func() { <-ctx.Done(); cc.Stop() }() - }) - return bound + info := c.CachedInfo() + if info != nil && info.Config.AckWait > 0 { + ackWait.Store(int64(info.Config.AckWait)) + } + pending := 1000 + if info != nil && info.Config.MaxAckPending > 0 { + pending = info.Config.MaxAckPending + } + cc, err := c.Consume(func(m jetstream.Msg) { q.take(ctx, m) }, jetstream.PullMaxMessages(pending)) + if err != nil { + q.logf("the seat %s's work queue %s is not pulled: %v", meshbuilds.Seat, worker, err) + return false + } + pulling = cc + q.logf("pulling the seat %s's asks from %s", meshbuilds.Seat, worker) + return true } - if bind() { - return nil + release := func() { + mu.Lock() + defer mu.Unlock() + if pulling == nil { + return + } + pulling.Stop() + pulling = nil + q.logf("the seat %s's work queue %s on %s was deleted: its asks wait on it, and it is pulled again when the bus says it was created", meshbuilds.Seat, worker, stream) } - q.logf("the seat %s's work queue %s on %s is not there yet: its asks wait on it, and it is pulled when the bus says it was created", meshbuilds.Seat, worker, stream) - sub, err := conn.Subscribe("$JS.EVENT.ADVISORY.CONSUMER.CREATED."+stream+"."+worker, func(*nats.Msg) { go bind() }) + created, err := conn.Subscribe("$JS.EVENT.ADVISORY.CONSUMER.CREATED."+stream+"."+worker, func(*nats.Msg) { go bind() }) if err != nil { return fmt.Errorf("hearing the creation of the seat %s's work queue: %w", meshbuilds.Seat, err) } - go func() { <-ctx.Done(); _ = sub.Unsubscribe() }() - if bind() { // made between the first look and the subscription - _ = sub.Unsubscribe() + deleted, err := conn.Subscribe("$JS.EVENT.ADVISORY.CONSUMER.DELETED."+stream+"."+worker, func(*nats.Msg) { + go func() { + release() + bind() // made again before the deletion was heard + }() + }) + if err != nil { + _ = created.Unsubscribe() + return fmt.Errorf("hearing the deletion of the seat %s's work queue: %w", meshbuilds.Seat, err) + } + go func() { + <-ctx.Done() + _ = created.Unsubscribe() + _ = deleted.Unsubscribe() + release() + }() + if !bind() { + q.logf("the seat %s's work queue %s on %s is not there yet: its asks wait on it, and it is pulled when the bus says it was created", meshbuilds.Seat, worker, stream) } return nil }