Builder: read the last versions-filled at start
The builder kept the catalog's versions-filled in memory only and its durable consumer took the event once, so a restart left every build wait-held. It now reads the last message of the subject from the events stream at its start (a read that fails is retried with backoff in the background). Builds: design 47 G A-408, §369, §372
This commit is contained in:
@@ -0,0 +1,82 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
)
|
||||
|
||||
// lastKept reads the last message kept on a subject: found false when the stream or the message is not there.
|
||||
type lastKept func(ctx context.Context, subject string) (found bool, err error)
|
||||
|
||||
// lastKeptOf reads from the stream the events are kept on, by a direct read of the last message of the subject.
|
||||
func lastKeptOf(js jetstream.JetStream, stream string) lastKept {
|
||||
return func(ctx context.Context, subject string) (bool, error) {
|
||||
s, err := js.Stream(ctx, stream)
|
||||
if err != nil {
|
||||
if errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
if _, err := s.GetLastMsgForSubject(ctx, subject); err != nil {
|
||||
if errors.Is(err, jetstream.ErrMsgNotFound) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
|
||||
// readFilled reads at the builder's start the last versions-filled the stream keeps (design 47 G §369, §372: kept with
|
||||
// no age limit) and takes it as heard; the event heard later sets it as before. Nothing found leaves it unset, said; a
|
||||
// read that failed is said and tried again in the background with backoff, never holding the start.
|
||||
func (rv *reservations) readFilled(ctx context.Context, read lastKept, wait func(context.Context, time.Duration) bool) {
|
||||
const first, most = time.Second, time.Minute
|
||||
try := func() (done bool, err error) {
|
||||
found, err := read(ctx, meshcatalog.SubjectVersionsFilled)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !found {
|
||||
rv.logf("the stream keeps no versions-filled yet (read at start): versions are reserved once the catalog says it")
|
||||
return true, nil
|
||||
}
|
||||
rv.filled.Store(true)
|
||||
rv.logf("the catalog said versions-filled (read at start): versions are reserved from now on")
|
||||
if rv.wake != nil {
|
||||
rv.wake(ctx)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
if done, err := try(); done {
|
||||
return
|
||||
} else {
|
||||
rv.logf("the last versions-filled is not read at start: %v; read again in the background", err)
|
||||
}
|
||||
go func() {
|
||||
for d := first; ; d = min(d*2, most) {
|
||||
if !wait(ctx, d) {
|
||||
return
|
||||
}
|
||||
done, err := try()
|
||||
if done {
|
||||
return
|
||||
}
|
||||
rv.logf("the last versions-filled is not read: %v; read again in %s", err, min(d*2, most))
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// sleeps waits d, as a context's bound (ADR 0437 §8: the builder holds no timer); false when ctx ended first.
|
||||
func sleeps(ctx context.Context, d time.Duration) bool {
|
||||
bound, cancel := context.WithTimeout(ctx, d)
|
||||
defer cancel()
|
||||
<-bound.Done()
|
||||
return ctx.Err() == nil
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestTheLastVersionsFilledIsReadAtStart(t *testing.T) {
|
||||
boom := errors.New("bus down")
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
results []error // per read: nil found, boom fails
|
||||
notFound bool
|
||||
wantFilled bool
|
||||
wantReads int
|
||||
waits []time.Duration
|
||||
}{
|
||||
{"kept", []error{nil}, false, true, 1, nil},
|
||||
{"none kept", []error{nil}, true, false, 1, nil},
|
||||
{"read fails then kept", []error{boom, boom, nil}, false, true, 3, []time.Duration{time.Second, 2 * time.Second}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
rv := newReservations(nil, nil, nil, nil, nil, t.Logf)
|
||||
var mu sync.Mutex
|
||||
woke := make(chan struct{}, 1)
|
||||
rv.wake = func(context.Context) { woke <- struct{}{} }
|
||||
reads := 0
|
||||
read := func(_ context.Context, subject string) (bool, error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
err := tc.results[reads]
|
||||
reads++
|
||||
return !tc.notFound, err
|
||||
}
|
||||
var waits []time.Duration
|
||||
wait := func(_ context.Context, d time.Duration) bool {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
waits = append(waits, d)
|
||||
return true
|
||||
}
|
||||
rv.readFilled(context.Background(), read, wait)
|
||||
if tc.wantFilled {
|
||||
select {
|
||||
case <-woke:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the read did not take the fill")
|
||||
}
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if rv.Filled() != tc.wantFilled || reads != tc.wantReads {
|
||||
t.Fatalf("filled %v reads %d", rv.Filled(), reads)
|
||||
}
|
||||
if !reflect.DeepEqual(waits, tc.waits) {
|
||||
t.Fatalf("backoff %v, want %v", waits, tc.waits)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -226,6 +226,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return emitSeatEvent(context.WithoutCancel(ctx), js, cred, meshbuilds.SubjectPackagePublished, "package-published-"+p.Module+"-"+p.Version+"-"+p.Artifact, p)
|
||||
}
|
||||
b.Reserved = reserved
|
||||
// The last versions-filled the stream keeps is read at the start (design 47 G §369, §372): a restart does not lose it.
|
||||
reserved.readFilled(ctx, lastKeptOf(js, eventsStream), sleeps)
|
||||
// A state bound later numbers the builds looked at without it.
|
||||
versionsKV.Ready(func(jetstream.KeyValue) { b.Reconsider(ctx) })
|
||||
// The versions' hand acts (builder_publish_again, builder_floor_again) come through reasks, as builder_build_again.
|
||||
|
||||
Reference in New Issue
Block a user