Builder: read the last versions-filled at start

The builder kept the catalog's versions-filled in memory only and its durable
consumer took the event once, so a restart left every build wait-held. It now
reads the last message of the subject from the events stream at its start (a
read that fails is retried with backoff in the background).

Builds: design 47 G A-408, §369, §372
This commit is contained in:
2026-11-14 22:49:04 +01:00
parent 1b3cc20127
commit 0071181fb1
3 changed files with 148 additions and 0 deletions
@@ -0,0 +1,82 @@
package main
import (
"context"
"errors"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
)
// lastKept reads the last message kept on a subject: found false when the stream or the message is not there.
type lastKept func(ctx context.Context, subject string) (found bool, err error)
// lastKeptOf reads from the stream the events are kept on, by a direct read of the last message of the subject.
func lastKeptOf(js jetstream.JetStream, stream string) lastKept {
return func(ctx context.Context, subject string) (bool, error) {
s, err := js.Stream(ctx, stream)
if err != nil {
if errors.Is(err, jetstream.ErrStreamNotFound) {
return false, nil
}
return false, err
}
if _, err := s.GetLastMsgForSubject(ctx, subject); err != nil {
if errors.Is(err, jetstream.ErrMsgNotFound) {
return false, nil
}
return false, err
}
return true, nil
}
}
// readFilled reads at the builder's start the last versions-filled the stream keeps (design 47 G §369, §372: kept with
// no age limit) and takes it as heard; the event heard later sets it as before. Nothing found leaves it unset, said; a
// read that failed is said and tried again in the background with backoff, never holding the start.
func (rv *reservations) readFilled(ctx context.Context, read lastKept, wait func(context.Context, time.Duration) bool) {
const first, most = time.Second, time.Minute
try := func() (done bool, err error) {
found, err := read(ctx, meshcatalog.SubjectVersionsFilled)
if err != nil {
return false, err
}
if !found {
rv.logf("the stream keeps no versions-filled yet (read at start): versions are reserved once the catalog says it")
return true, nil
}
rv.filled.Store(true)
rv.logf("the catalog said versions-filled (read at start): versions are reserved from now on")
if rv.wake != nil {
rv.wake(ctx)
}
return true, nil
}
if done, err := try(); done {
return
} else {
rv.logf("the last versions-filled is not read at start: %v; read again in the background", err)
}
go func() {
for d := first; ; d = min(d*2, most) {
if !wait(ctx, d) {
return
}
done, err := try()
if done {
return
}
rv.logf("the last versions-filled is not read: %v; read again in %s", err, min(d*2, most))
}
}()
}
// sleeps waits d, as a context's bound (ADR 0437 §8: the builder holds no timer); false when ctx ended first.
func sleeps(ctx context.Context, d time.Duration) bool {
bound, cancel := context.WithTimeout(ctx, d)
defer cancel()
<-bound.Done()
return ctx.Err() == nil
}
@@ -0,0 +1,64 @@
package main
import (
"context"
"errors"
"reflect"
"sync"
"testing"
"time"
)
func TestTheLastVersionsFilledIsReadAtStart(t *testing.T) {
boom := errors.New("bus down")
for _, tc := range []struct {
name string
results []error // per read: nil found, boom fails
notFound bool
wantFilled bool
wantReads int
waits []time.Duration
}{
{"kept", []error{nil}, false, true, 1, nil},
{"none kept", []error{nil}, true, false, 1, nil},
{"read fails then kept", []error{boom, boom, nil}, false, true, 3, []time.Duration{time.Second, 2 * time.Second}},
} {
t.Run(tc.name, func(t *testing.T) {
rv := newReservations(nil, nil, nil, nil, nil, t.Logf)
var mu sync.Mutex
woke := make(chan struct{}, 1)
rv.wake = func(context.Context) { woke <- struct{}{} }
reads := 0
read := func(_ context.Context, subject string) (bool, error) {
mu.Lock()
defer mu.Unlock()
err := tc.results[reads]
reads++
return !tc.notFound, err
}
var waits []time.Duration
wait := func(_ context.Context, d time.Duration) bool {
mu.Lock()
defer mu.Unlock()
waits = append(waits, d)
return true
}
rv.readFilled(context.Background(), read, wait)
if tc.wantFilled {
select {
case <-woke:
case <-time.After(5 * time.Second):
t.Fatal("the read did not take the fill")
}
}
mu.Lock()
defer mu.Unlock()
if rv.Filled() != tc.wantFilled || reads != tc.wantReads {
t.Fatalf("filled %v reads %d", rv.Filled(), reads)
}
if !reflect.DeepEqual(waits, tc.waits) {
t.Fatalf("backoff %v, want %v", waits, tc.waits)
}
})
}
}
@@ -226,6 +226,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return emitSeatEvent(context.WithoutCancel(ctx), js, cred, meshbuilds.SubjectPackagePublished, "package-published-"+p.Module+"-"+p.Version+"-"+p.Artifact, p)
}
b.Reserved = reserved
// The last versions-filled the stream keeps is read at the start (design 47 G §369, §372): a restart does not lose it.
reserved.readFilled(ctx, lastKeptOf(js, eventsStream), sleeps)
// A state bound later numbers the builds looked at without it.
versionsKV.Ready(func(jetstream.KeyValue) { b.Reconsider(ctx) })
// The versions' hand acts (builder_publish_again, builder_floor_again) come through reasks, as builder_build_again.