From 0071181fb14593b0400ea2c1b79bf34c867a319f Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 14 Nov 2026 22:49:04 +0100 Subject: [PATCH] Builder: read the last versions-filled at start MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The builder kept the catalog's versions-filled in memory only and its durable consumer took the event once, so a restart left every build wait-held. It now reads the last message of the subject from the events stream at its start (a read that fails is retried with backoff in the background). Builds: design 47 G A-408, §369, §372 --- .../cmd/mesh-delivery-builder/filledread.go | 82 +++++++++++++++++++ .../mesh-delivery-builder/filledread_test.go | 64 +++++++++++++++ .../cmd/mesh-delivery-builder/main.go | 2 + 3 files changed, 148 insertions(+) create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder/filledread.go create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder/filledread_test.go diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/filledread.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/filledread.go new file mode 100644 index 00000000..d111d521 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/filledread.go @@ -0,0 +1,82 @@ +package main + +import ( + "context" + "errors" + "time" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" +) + +// lastKept reads the last message kept on a subject: found false when the stream or the message is not there. +type lastKept func(ctx context.Context, subject string) (found bool, err error) + +// lastKeptOf reads from the stream the events are kept on, by a direct read of the last message of the subject. +func lastKeptOf(js jetstream.JetStream, stream string) lastKept { + return func(ctx context.Context, subject string) (bool, error) { + s, err := js.Stream(ctx, stream) + if err != nil { + if errors.Is(err, jetstream.ErrStreamNotFound) { + return false, nil + } + return false, err + } + if _, err := s.GetLastMsgForSubject(ctx, subject); err != nil { + if errors.Is(err, jetstream.ErrMsgNotFound) { + return false, nil + } + return false, err + } + return true, nil + } +} + +// readFilled reads at the builder's start the last versions-filled the stream keeps (design 47 G §369, §372: kept with +// no age limit) and takes it as heard; the event heard later sets it as before. Nothing found leaves it unset, said; a +// read that failed is said and tried again in the background with backoff, never holding the start. +func (rv *reservations) readFilled(ctx context.Context, read lastKept, wait func(context.Context, time.Duration) bool) { + const first, most = time.Second, time.Minute + try := func() (done bool, err error) { + found, err := read(ctx, meshcatalog.SubjectVersionsFilled) + if err != nil { + return false, err + } + if !found { + rv.logf("the stream keeps no versions-filled yet (read at start): versions are reserved once the catalog says it") + return true, nil + } + rv.filled.Store(true) + rv.logf("the catalog said versions-filled (read at start): versions are reserved from now on") + if rv.wake != nil { + rv.wake(ctx) + } + return true, nil + } + if done, err := try(); done { + return + } else { + rv.logf("the last versions-filled is not read at start: %v; read again in the background", err) + } + go func() { + for d := first; ; d = min(d*2, most) { + if !wait(ctx, d) { + return + } + done, err := try() + if done { + return + } + rv.logf("the last versions-filled is not read: %v; read again in %s", err, min(d*2, most)) + } + }() +} + +// sleeps waits d, as a context's bound (ADR 0437 §8: the builder holds no timer); false when ctx ended first. +func sleeps(ctx context.Context, d time.Duration) bool { + bound, cancel := context.WithTimeout(ctx, d) + defer cancel() + <-bound.Done() + return ctx.Err() == nil +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/filledread_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/filledread_test.go new file mode 100644 index 00000000..fef97a57 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/filledread_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "context" + "errors" + "reflect" + "sync" + "testing" + "time" +) + +func TestTheLastVersionsFilledIsReadAtStart(t *testing.T) { + boom := errors.New("bus down") + for _, tc := range []struct { + name string + results []error // per read: nil found, boom fails + notFound bool + wantFilled bool + wantReads int + waits []time.Duration + }{ + {"kept", []error{nil}, false, true, 1, nil}, + {"none kept", []error{nil}, true, false, 1, nil}, + {"read fails then kept", []error{boom, boom, nil}, false, true, 3, []time.Duration{time.Second, 2 * time.Second}}, + } { + t.Run(tc.name, func(t *testing.T) { + rv := newReservations(nil, nil, nil, nil, nil, t.Logf) + var mu sync.Mutex + woke := make(chan struct{}, 1) + rv.wake = func(context.Context) { woke <- struct{}{} } + reads := 0 + read := func(_ context.Context, subject string) (bool, error) { + mu.Lock() + defer mu.Unlock() + err := tc.results[reads] + reads++ + return !tc.notFound, err + } + var waits []time.Duration + wait := func(_ context.Context, d time.Duration) bool { + mu.Lock() + defer mu.Unlock() + waits = append(waits, d) + return true + } + rv.readFilled(context.Background(), read, wait) + if tc.wantFilled { + select { + case <-woke: + case <-time.After(5 * time.Second): + t.Fatal("the read did not take the fill") + } + } + mu.Lock() + defer mu.Unlock() + if rv.Filled() != tc.wantFilled || reads != tc.wantReads { + t.Fatalf("filled %v reads %d", rv.Filled(), reads) + } + if !reflect.DeepEqual(waits, tc.waits) { + t.Fatalf("backoff %v, want %v", waits, tc.waits) + } + }) + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 01f3195d..a37cb65f 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -226,6 +226,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return emitSeatEvent(context.WithoutCancel(ctx), js, cred, meshbuilds.SubjectPackagePublished, "package-published-"+p.Module+"-"+p.Version+"-"+p.Artifact, p) } b.Reserved = reserved + // The last versions-filled the stream keeps is read at the start (design 47 G §369, §372): a restart does not lose it. + reserved.readFilled(ctx, lastKeptOf(js, eventsStream), sleeps) // A state bound later numbers the builds looked at without it. versionsKV.Ready(func(jetstream.KeyValue) { b.Reconsider(ctx) }) // The versions' hand acts (builder_publish_again, builder_floor_again) come through reasks, as builder_build_again.