H-A-251: bind a state's bucket once for the process, not before every read

The runtime asked the bucket's stream info before every get, put, delete and
listing, so each state read was two answers on the bus (a third of the night's
flood came this way, ten thousand reads per sweep). A Conn is one account's
connection; it now keeps one binding per bucket, shared by concurrent askers,
and binds again once when the bus says the bucket is gone.

Builds: design 47 H H-A-251
This commit is contained in:
2026-11-21 11:55:35 +01:00
parent fe707779e3
commit 05d8f6b762
5 changed files with 346 additions and 51 deletions
+141
View File
@@ -0,0 +1,141 @@
package bus
import (
"context"
"encoding/json"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/nats-io/nats.go/jetstream"
)
// boundKV is a bucket's binding in a test: it answers every get with one value, or, once gone, as the bus answers a
// binding whose bucket was deleted.
type boundKV struct {
jetstream.KeyValue
gone atomic.Bool
puts atomic.Int32
}
type boundEntry struct {
jetstream.KeyValueEntry
key string
}
func (e boundEntry) Key() string { return e.key }
func (e boundEntry) Value() []byte { return []byte(`{"v":1}`) }
func (e boundEntry) Revision() uint64 { return 1 }
func (boundEntry) Created() time.Time { return time.Time{} }
func (*boundKV) Bucket() string { return "gitea_pins" }
func (kv *boundKV) Get(_ context.Context, key string) (jetstream.KeyValueEntry, error) {
if kv.gone.Load() {
return nil, jetstream.ErrStreamNotFound
}
return boundEntry{key: key}, nil
}
func (kv *boundKV) Put(context.Context, string, []byte) (uint64, error) {
if kv.gone.Load() {
return 0, jetstream.ErrNoStreamResponse
}
return uint64(kv.puts.Add(1)), nil
}
// **A state is bound once for the process, and bound again only when the bus says its bucket is gone** (design 47 H
// lead answers H-A-251): binding is a stream-info request on the bus, and asked before every get it made each read two
// answers on the bus.
func TestAStateIsBoundOnceAndBoundAgainWhenItsBucketIsGone(t *testing.T) {
setup := func() (*Conn, *atomic.Int32, *[]*boundKV) {
var binds atomic.Int32
var mu sync.Mutex
made := []*boundKV{}
c := &Conn{node: "anchor", issued: map[string]*Membership{"gitea": {Module: "gitea",
State: []StateIssued{{Name: "pins", Bucket: "gitea_pins", Writes: true}}}}}
c.bind = func(_ context.Context, bucket string) (jetstream.KeyValue, error) {
binds.Add(1)
if bucket != "gitea_pins" {
t.Errorf("bound %s, want gitea_pins", bucket)
}
kv := &boundKV{}
mu.Lock()
made = append(made, kv)
mu.Unlock()
return kv, nil
}
return c, &binds, &made
}
t.Run("two reads, one bind", func(t *testing.T) {
c, binds, _ := setup()
for range 2 {
if e, err := c.StateGet("gitea", "pins", "a"); err != nil || e == nil || e.Key != "a" {
t.Fatalf("read: %+v, %v", e, err)
}
}
if _, err := c.StatePut("gitea", "pins", "a", json.RawMessage(`{"v":2}`)); err != nil {
t.Fatal(err)
}
if got := binds.Load(); got != 1 {
t.Fatalf("two reads and a write bound the bucket %d times, want once", got)
}
})
t.Run("many readers at once, one bind", func(t *testing.T) {
c, binds, _ := setup()
var wg sync.WaitGroup
for range 16 {
wg.Go(func() {
if _, err := c.StateGet("gitea", "pins", "a"); err != nil {
t.Error(err)
}
})
}
wg.Wait()
if got := binds.Load(); got != 1 {
t.Fatalf("16 readers bound the bucket %d times, want once", got)
}
})
t.Run("a bucket gone under the runner is bound again once", func(t *testing.T) {
c, binds, made := setup()
if _, err := c.StateGet("gitea", "pins", "a"); err != nil {
t.Fatal(err)
}
(*made)[0].gone.Store(true)
if e, err := c.StateGet("gitea", "pins", "a"); err != nil || e == nil {
t.Fatalf("the read after the bucket was made again: %+v, %v", e, err)
}
if got := binds.Load(); got != 2 {
t.Fatalf("the bucket was bound %d times, want twice (once more after it was gone)", got)
}
if _, err := c.StateGet("gitea", "pins", "a"); err != nil {
t.Fatal(err)
}
(*made)[1].gone.Store(true)
if _, err := c.StatePut("gitea", "pins", "a", json.RawMessage(`{"v":2}`)); err != nil {
t.Fatal(err)
}
if got, puts := binds.Load(), (*made)[2].puts.Load(); got != 3 || puts != 1 {
t.Fatalf("a write to a gone bucket bound it %d times in all (want 3) and wrote %d times on the new binding (want 1)", got, puts)
}
})
t.Run("a bucket still gone after the bind again is said", func(t *testing.T) {
c, binds, made := setup()
if _, err := c.StateGet("gitea", "pins", "a"); err != nil {
t.Fatal(err)
}
(*made)[0].gone.Store(true)
c.bind = func(context.Context, string) (jetstream.KeyValue, error) {
binds.Add(1)
return nil, jetstream.ErrBucketNotFound
}
if _, err := c.StateGet("gitea", "pins", "a"); err == nil {
t.Fatal("a read of a bucket no longer on the bus answered no error")
}
if got := binds.Load(); got != 2 {
t.Fatalf("the bucket was bound %d times, want twice: the bind again is once", got)
}
})
}