H-A-251: bind a state's bucket once for the process, not before every read
The runtime asked the bucket's stream info before every get, put, delete and listing, so each state read was two answers on the bus (a third of the night's flood came this way, ten thousand reads per sweep). A Conn is one account's connection; it now keeps one binding per bucket, shared by concurrent askers, and binds again once when the bus says the bucket is gone. Builds: design 47 H H-A-251
This commit is contained in:
@@ -15,6 +15,7 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/membership"
|
||||
"git.novox.be/novox/mesh-sdk/go/pernode"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
@@ -92,20 +93,107 @@ func (c *Conn) issuedState(module, name string) (StateIssued, error) {
|
||||
"issued: %s (novox/hq ADR 0201, ADR 0595 §1)", module, name, issued)
|
||||
}
|
||||
|
||||
// binding is one bucket's binding, or the bind in flight for it: the first asker binds, an asker meanwhile waits for
|
||||
// that bind (done) instead of asking the bus again.
|
||||
type binding struct {
|
||||
done chan struct{}
|
||||
kv jetstream.KeyValue
|
||||
err error
|
||||
}
|
||||
|
||||
// bucket is the state's binding, bound once for the process (design 47 H lead answers H-A-251): binding a bucket is a
|
||||
// stream-info request on the bus, and asked before every get it made each read two answers. A bind that fails is not
|
||||
// kept, so the next ask binds again.
|
||||
func (c *Conn) bucket(ctx context.Context, s StateIssued) (jetstream.KeyValue, error) {
|
||||
c.bindMu.Lock()
|
||||
b := c.bound[s.Bucket]
|
||||
if b == nil {
|
||||
if c.bound == nil {
|
||||
c.bound = map[string]*binding{}
|
||||
}
|
||||
b = &binding{done: make(chan struct{})}
|
||||
c.bound[s.Bucket] = b
|
||||
c.bindMu.Unlock()
|
||||
b.kv, b.err = c.bindBucket(ctx, s.Bucket)
|
||||
if b.err != nil {
|
||||
c.bindMu.Lock()
|
||||
if c.bound[s.Bucket] == b {
|
||||
delete(c.bound, s.Bucket)
|
||||
}
|
||||
c.bindMu.Unlock()
|
||||
}
|
||||
close(b.done)
|
||||
} else {
|
||||
c.bindMu.Unlock()
|
||||
select {
|
||||
case <-b.done:
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
if b.err != nil {
|
||||
if errors.Is(b.err, jetstream.ErrBucketNotFound) {
|
||||
return nil, fmt.Errorf("the state %q is issued and its bucket is not on the bus yet: the mesh "+
|
||||
"creates it from the catalogue on its next raise", s.Name)
|
||||
}
|
||||
return nil, b.err
|
||||
}
|
||||
return b.kv, nil
|
||||
}
|
||||
|
||||
func (c *Conn) bindBucket(ctx context.Context, bucket string) (jetstream.KeyValue, error) {
|
||||
if c.bind != nil {
|
||||
return c.bind(ctx, bucket)
|
||||
}
|
||||
js, err := jetstream.New(c.nc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kv, err := js.KeyValue(ctx, s.Bucket)
|
||||
if err != nil {
|
||||
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
return nil, fmt.Errorf("the state %q is issued and its bucket is not on the bus yet: the mesh "+
|
||||
"creates it from the catalogue on its next raise", s.Name)
|
||||
}
|
||||
return nil, err
|
||||
return js.KeyValue(ctx, bucket)
|
||||
}
|
||||
|
||||
// unbind drops a bucket's binding the bus said is gone, when it is still the one kept, so the next ask binds again.
|
||||
func (c *Conn) unbind(bucket string, kv jetstream.KeyValue) {
|
||||
c.bindMu.Lock()
|
||||
defer c.bindMu.Unlock()
|
||||
b := c.bound[bucket]
|
||||
if b == nil {
|
||||
return
|
||||
}
|
||||
return kv, nil
|
||||
select {
|
||||
case <-b.done: // bound: its kv is written
|
||||
default:
|
||||
return // bound again already, in flight
|
||||
}
|
||||
if b.kv == kv {
|
||||
delete(c.bound, bucket)
|
||||
}
|
||||
}
|
||||
|
||||
// bucketGone is the bus saying a kept binding's bucket is not there (deleted, or deleted and created again): no stream of
|
||||
// the name, or nobody answers on its subjects.
|
||||
func bucketGone(err error) bool {
|
||||
return errors.Is(err, jetstream.ErrStreamNotFound) || errors.Is(err, jetstream.ErrBucketNotFound) ||
|
||||
errors.Is(err, jetstream.ErrNoStreamResponse) || errors.Is(err, nats.ErrNoResponders)
|
||||
}
|
||||
|
||||
// inBucket does one act on the state's kept binding; when the bus says its bucket is gone, the binding is dropped, the
|
||||
// bucket bound again once and the act done once more on the new binding (H-A-251).
|
||||
func inBucket[T any](c *Conn, ctx context.Context, s StateIssued, do func(jetstream.KeyValue) (T, error)) (T, error) {
|
||||
var none T
|
||||
kv, err := c.bucket(ctx, s)
|
||||
if err != nil {
|
||||
return none, err
|
||||
}
|
||||
got, err := do(kv)
|
||||
if !bucketGone(err) {
|
||||
return got, err
|
||||
}
|
||||
c.unbind(s.Bucket, kv)
|
||||
if kv, err = c.bucket(ctx, s); err != nil {
|
||||
return none, err
|
||||
}
|
||||
return do(kv)
|
||||
}
|
||||
|
||||
func checkKey(key string) error {
|
||||
@@ -130,11 +218,7 @@ func (c *Conn) StateGet(module, name, key string) (*StateEntry, error) {
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
defer cancel()
|
||||
kv, err := c.bucket(ctx, s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
e, err := kv.Get(ctx, key)
|
||||
e, err := inBucket(c, ctx, s, func(kv jetstream.KeyValue) (jetstream.KeyValueEntry, error) { return kv.Get(ctx, key) })
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -210,11 +294,10 @@ func (c *Conn) StatePage(module, name, after string, maxBytes int, budget time.D
|
||||
maps.Copy(current, got)
|
||||
}
|
||||
} else {
|
||||
kv, err := c.bucket(ctx, s)
|
||||
if err != nil {
|
||||
return StatePage{}, err
|
||||
}
|
||||
if current, err = currentOf(ctx, kv); err != nil {
|
||||
var err error
|
||||
if current, err = inBucket(c, ctx, s, func(kv jetstream.KeyValue) (map[string]*StateEntry, error) {
|
||||
return currentOf(ctx, kv)
|
||||
}); err != nil {
|
||||
return StatePage{}, err
|
||||
}
|
||||
}
|
||||
@@ -273,11 +356,9 @@ func (c *Conn) StateHistory(module, name, key string) ([]StateKept, error) {
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
defer cancel()
|
||||
kv, err := c.bucket(ctx, s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entries, err := kv.History(ctx, key)
|
||||
entries, err := inBucket(c, ctx, s, func(kv jetstream.KeyValue) ([]jetstream.KeyValueEntry, error) {
|
||||
return kv.History(ctx, key)
|
||||
})
|
||||
kept := []StateKept{}
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return kept, nil
|
||||
@@ -326,11 +407,7 @@ func (c *Conn) StateInfo(module, name string) (*StateInfo, error) {
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
defer cancel()
|
||||
kv, err := c.bucket(ctx, s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
status, err := kv.Status(ctx)
|
||||
status, err := inBucket(c, ctx, s, func(kv jetstream.KeyValue) (jetstream.KeyValueStatus, error) { return kv.Status(ctx) })
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -363,11 +440,7 @@ func (c *Conn) StatePut(module, name, key string, value json.RawMessage) (uint64
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
defer cancel()
|
||||
kv, err := c.bucket(ctx, s)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return kv.Put(ctx, key, value)
|
||||
return inBucket(c, ctx, s, func(kv jetstream.KeyValue) (uint64, error) { return kv.Put(ctx, key, value) })
|
||||
}
|
||||
|
||||
// MinKeyTTL is the shortest time to live a key may be given: the bus counts a key's age in whole seconds (mesh-sdk
|
||||
@@ -437,11 +510,8 @@ func (c *Conn) StateDelete(module, name, key string) error {
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
defer cancel()
|
||||
kv, err := c.bucket(ctx, s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return kv.Delete(ctx, key)
|
||||
_, err = inBucket(c, ctx, s, func(kv jetstream.KeyValue) (struct{}, error) { return struct{}{}, kv.Delete(ctx, key) })
|
||||
return err
|
||||
}
|
||||
|
||||
// StateKeys is every key with a current value, sorted.
|
||||
@@ -455,11 +525,7 @@ func (c *Conn) StateKeys(module, name string) ([]string, error) {
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
defer cancel()
|
||||
kv, err := c.bucket(ctx, s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return keysOf(ctx, kv)
|
||||
return inBucket(c, ctx, s, func(kv jetstream.KeyValue) ([]string, error) { return keysOf(ctx, kv) })
|
||||
}
|
||||
|
||||
// keysOf is every key with a current value in one bucket, sorted.
|
||||
@@ -529,12 +595,16 @@ func (c *Conn) StateWatch(module, name, pattern string, deliver func(StateChange
|
||||
return c.perNodeWatch(module, s, pattern, deliver)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
kv, err := c.bucket(ctx, s)
|
||||
cancel()
|
||||
defer cancel()
|
||||
type watching struct{ stop func() }
|
||||
w, err := inBucket(c, ctx, s, func(kv jetstream.KeyValue) (watching, error) {
|
||||
stop, err := c.watchBucket(module, name, kv, filter, deliver)
|
||||
return watching{stop}, err
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return c.watchBucket(module, name, kv, filter, deliver)
|
||||
return w.stop, nil
|
||||
}
|
||||
|
||||
// watchBucket is StateWatch over one bucket: the current values, then every change.
|
||||
|
||||
Reference in New Issue
Block a user