From 05d8f6b762f074d0c1b8dc438d6250ade4c84186 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 11:55:35 +0100 Subject: [PATCH] H-A-251: bind a state's bucket once for the process, not before every read The runtime asked the bucket's stream info before every get, put, delete and listing, so each state read was two answers on the bus (a third of the night's flood came this way, ten thousand reads per sweep). A Conn is one account's connection; it now keeps one binding per bucket, shared by concurrent askers, and binds again once when the bus says the bucket is gone. Builds: design 47 H H-A-251 --- node-tools/internal/bus/bus.go | 7 + node-tools/internal/bus/seat.go | 6 +- node-tools/internal/bus/state.go | 162 +++++++++++++----- .../bus/state_bind_integration_test.go | 81 +++++++++ node-tools/internal/bus/state_bind_test.go | 141 +++++++++++++++ 5 files changed, 346 insertions(+), 51 deletions(-) create mode 100644 node-tools/internal/bus/state_bind_integration_test.go create mode 100644 node-tools/internal/bus/state_bind_test.go diff --git a/node-tools/internal/bus/bus.go b/node-tools/internal/bus/bus.go index 726dc0c..c13a122 100644 --- a/node-tools/internal/bus/bus.go +++ b/node-tools/internal/bus/bus.go @@ -203,6 +203,13 @@ type Conn struct { waits map[string]map[chan string]bool // inboxSaid is when a refused answer publish was last said, by subject (answerRefused). inboxSaid map[string]time.Time + // bound is each state's binding, kept for the process and bound again only when the bus says its bucket is gone + // (state.go, design 47 H lead answers H-A-251): one Conn is one account's connection, so a binding is per account + // and bucket. + bindMu sync.Mutex + bound map[string]*binding + // bind binds one bucket: jetstream's KeyValue (a stream-info request on the bus) unless a test replaces it. + bind func(ctx context.Context, bucket string) (jetstream.KeyValue, error) } // answered is one subject the runtime answers on: what to subscribe again with, and how often it was diff --git a/node-tools/internal/bus/seat.go b/node-tools/internal/bus/seat.go index 1ef7fc1..f703d16 100644 --- a/node-tools/internal/bus/seat.go +++ b/node-tools/internal/bus/seat.go @@ -427,11 +427,7 @@ func (c *Conn) stateWrite(module, name, key string, value json.RawMessage, } ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) defer cancel() - kv, err := c.bucket(ctx, s) - if err != nil { - return 0, err - } - rev, err := write(ctx, kv) + rev, err := inBucket(c, ctx, s, func(kv jetstream.KeyValue) (uint64, error) { return write(ctx, kv) }) if errors.Is(err, jetstream.ErrKeyExists) || isWrongSequence(err) { return 0, changed{name + "." + key} } diff --git a/node-tools/internal/bus/state.go b/node-tools/internal/bus/state.go index 1e482f9..a68e9ce 100644 --- a/node-tools/internal/bus/state.go +++ b/node-tools/internal/bus/state.go @@ -15,6 +15,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/membership" "git.novox.be/novox/mesh-sdk/go/pernode" + "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" ) @@ -92,20 +93,107 @@ func (c *Conn) issuedState(module, name string) (StateIssued, error) { "issued: %s (novox/hq ADR 0201, ADR 0595 ยง1)", module, name, issued) } +// binding is one bucket's binding, or the bind in flight for it: the first asker binds, an asker meanwhile waits for +// that bind (done) instead of asking the bus again. +type binding struct { + done chan struct{} + kv jetstream.KeyValue + err error +} + +// bucket is the state's binding, bound once for the process (design 47 H lead answers H-A-251): binding a bucket is a +// stream-info request on the bus, and asked before every get it made each read two answers. A bind that fails is not +// kept, so the next ask binds again. func (c *Conn) bucket(ctx context.Context, s StateIssued) (jetstream.KeyValue, error) { + c.bindMu.Lock() + b := c.bound[s.Bucket] + if b == nil { + if c.bound == nil { + c.bound = map[string]*binding{} + } + b = &binding{done: make(chan struct{})} + c.bound[s.Bucket] = b + c.bindMu.Unlock() + b.kv, b.err = c.bindBucket(ctx, s.Bucket) + if b.err != nil { + c.bindMu.Lock() + if c.bound[s.Bucket] == b { + delete(c.bound, s.Bucket) + } + c.bindMu.Unlock() + } + close(b.done) + } else { + c.bindMu.Unlock() + select { + case <-b.done: + case <-ctx.Done(): + return nil, ctx.Err() + } + } + if b.err != nil { + if errors.Is(b.err, jetstream.ErrBucketNotFound) { + return nil, fmt.Errorf("the state %q is issued and its bucket is not on the bus yet: the mesh "+ + "creates it from the catalogue on its next raise", s.Name) + } + return nil, b.err + } + return b.kv, nil +} + +func (c *Conn) bindBucket(ctx context.Context, bucket string) (jetstream.KeyValue, error) { + if c.bind != nil { + return c.bind(ctx, bucket) + } js, err := jetstream.New(c.nc) if err != nil { return nil, err } - kv, err := js.KeyValue(ctx, s.Bucket) - if err != nil { - if errors.Is(err, jetstream.ErrBucketNotFound) { - return nil, fmt.Errorf("the state %q is issued and its bucket is not on the bus yet: the mesh "+ - "creates it from the catalogue on its next raise", s.Name) - } - return nil, err + return js.KeyValue(ctx, bucket) +} + +// unbind drops a bucket's binding the bus said is gone, when it is still the one kept, so the next ask binds again. +func (c *Conn) unbind(bucket string, kv jetstream.KeyValue) { + c.bindMu.Lock() + defer c.bindMu.Unlock() + b := c.bound[bucket] + if b == nil { + return } - return kv, nil + select { + case <-b.done: // bound: its kv is written + default: + return // bound again already, in flight + } + if b.kv == kv { + delete(c.bound, bucket) + } +} + +// bucketGone is the bus saying a kept binding's bucket is not there (deleted, or deleted and created again): no stream of +// the name, or nobody answers on its subjects. +func bucketGone(err error) bool { + return errors.Is(err, jetstream.ErrStreamNotFound) || errors.Is(err, jetstream.ErrBucketNotFound) || + errors.Is(err, jetstream.ErrNoStreamResponse) || errors.Is(err, nats.ErrNoResponders) +} + +// inBucket does one act on the state's kept binding; when the bus says its bucket is gone, the binding is dropped, the +// bucket bound again once and the act done once more on the new binding (H-A-251). +func inBucket[T any](c *Conn, ctx context.Context, s StateIssued, do func(jetstream.KeyValue) (T, error)) (T, error) { + var none T + kv, err := c.bucket(ctx, s) + if err != nil { + return none, err + } + got, err := do(kv) + if !bucketGone(err) { + return got, err + } + c.unbind(s.Bucket, kv) + if kv, err = c.bucket(ctx, s); err != nil { + return none, err + } + return do(kv) } func checkKey(key string) error { @@ -130,11 +218,7 @@ func (c *Conn) StateGet(module, name, key string) (*StateEntry, error) { } ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) defer cancel() - kv, err := c.bucket(ctx, s) - if err != nil { - return nil, err - } - e, err := kv.Get(ctx, key) + e, err := inBucket(c, ctx, s, func(kv jetstream.KeyValue) (jetstream.KeyValueEntry, error) { return kv.Get(ctx, key) }) if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, nil } @@ -210,11 +294,10 @@ func (c *Conn) StatePage(module, name, after string, maxBytes int, budget time.D maps.Copy(current, got) } } else { - kv, err := c.bucket(ctx, s) - if err != nil { - return StatePage{}, err - } - if current, err = currentOf(ctx, kv); err != nil { + var err error + if current, err = inBucket(c, ctx, s, func(kv jetstream.KeyValue) (map[string]*StateEntry, error) { + return currentOf(ctx, kv) + }); err != nil { return StatePage{}, err } } @@ -273,11 +356,9 @@ func (c *Conn) StateHistory(module, name, key string) ([]StateKept, error) { } ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) defer cancel() - kv, err := c.bucket(ctx, s) - if err != nil { - return nil, err - } - entries, err := kv.History(ctx, key) + entries, err := inBucket(c, ctx, s, func(kv jetstream.KeyValue) ([]jetstream.KeyValueEntry, error) { + return kv.History(ctx, key) + }) kept := []StateKept{} if errors.Is(err, jetstream.ErrKeyNotFound) { return kept, nil @@ -326,11 +407,7 @@ func (c *Conn) StateInfo(module, name string) (*StateInfo, error) { } ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) defer cancel() - kv, err := c.bucket(ctx, s) - if err != nil { - return nil, err - } - status, err := kv.Status(ctx) + status, err := inBucket(c, ctx, s, func(kv jetstream.KeyValue) (jetstream.KeyValueStatus, error) { return kv.Status(ctx) }) if err != nil { return nil, err } @@ -363,11 +440,7 @@ func (c *Conn) StatePut(module, name, key string, value json.RawMessage) (uint64 } ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) defer cancel() - kv, err := c.bucket(ctx, s) - if err != nil { - return 0, err - } - return kv.Put(ctx, key, value) + return inBucket(c, ctx, s, func(kv jetstream.KeyValue) (uint64, error) { return kv.Put(ctx, key, value) }) } // MinKeyTTL is the shortest time to live a key may be given: the bus counts a key's age in whole seconds (mesh-sdk @@ -437,11 +510,8 @@ func (c *Conn) StateDelete(module, name, key string) error { } ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) defer cancel() - kv, err := c.bucket(ctx, s) - if err != nil { - return err - } - return kv.Delete(ctx, key) + _, err = inBucket(c, ctx, s, func(kv jetstream.KeyValue) (struct{}, error) { return struct{}{}, kv.Delete(ctx, key) }) + return err } // StateKeys is every key with a current value, sorted. @@ -455,11 +525,7 @@ func (c *Conn) StateKeys(module, name string) ([]string, error) { } ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) defer cancel() - kv, err := c.bucket(ctx, s) - if err != nil { - return nil, err - } - return keysOf(ctx, kv) + return inBucket(c, ctx, s, func(kv jetstream.KeyValue) ([]string, error) { return keysOf(ctx, kv) }) } // keysOf is every key with a current value in one bucket, sorted. @@ -529,12 +595,16 @@ func (c *Conn) StateWatch(module, name, pattern string, deliver func(StateChange return c.perNodeWatch(module, s, pattern, deliver) } ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) - kv, err := c.bucket(ctx, s) - cancel() + defer cancel() + type watching struct{ stop func() } + w, err := inBucket(c, ctx, s, func(kv jetstream.KeyValue) (watching, error) { + stop, err := c.watchBucket(module, name, kv, filter, deliver) + return watching{stop}, err + }) if err != nil { return nil, err } - return c.watchBucket(module, name, kv, filter, deliver) + return w.stop, nil } // watchBucket is StateWatch over one bucket: the current values, then every change. diff --git a/node-tools/internal/bus/state_bind_integration_test.go b/node-tools/internal/bus/state_bind_integration_test.go new file mode 100644 index 0000000..22e4a20 --- /dev/null +++ b/node-tools/internal/bus/state_bind_integration_test.go @@ -0,0 +1,81 @@ +//go:build integration + +package bus_test + +import ( + "strings" + "sync/atomic" + "testing" + + "github.com/nats-io/nats.go" + + mt "github.com/novox/mesh-tools/node-tools/internal/meshtest" +) + +// **On the bus, a read of a state is one answer: its bucket is bound once, and bound again when it was made afresh or +// is gone** (design 47 H lead answers H-A-251). The runtime asked the bucket's stream info before every get, so each +// read was two answers; a third of a night's bus flood came this way. +func TestAStateReadOnTheBusBindsItsBucketOnce(t *testing.T) { + conn, kv := owedWatcher(t) + const bucket = "mesh-delivery-assigner_pins" + if _, err := kv.Put("a", []byte(`{"v":1}`)); err != nil { + t.Fatal(err) + } + watcher, err := nats.Connect(mt.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(watcher.Close) + var infos atomic.Int32 + if _, err := watcher.Subscribe("$JS.API.STREAM.INFO.KV_"+bucket, func(*nats.Msg) { infos.Add(1) }); err != nil { + t.Fatal(err) + } + if err := watcher.Flush(); err != nil { + t.Fatal(err) + } + read := func() { + t.Helper() + e, err := conn.StateGet("gitea", "mesh-delivery-assigner.pins", "a") + if err != nil || e == nil { + t.Fatalf("read: %+v, %v", e, err) + } + } + read() + read() + read() + if err := watcher.Flush(); err != nil { + t.Fatal(err) + } + if got := infos.Load(); got != 1 { + t.Fatalf("three reads asked the bucket's stream info %d times, want once", got) + } + + // The bucket deleted under the runtime: the read binds again, once, and says the bucket is not on the bus. + js, _ := watcher.JetStream() + if err := js.DeleteKeyValue(bucket); err != nil { + t.Fatal(err) + } + _, err = conn.StateGet("gitea", "mesh-delivery-assigner.pins", "a") + if err == nil || !strings.Contains(err.Error(), "not on the bus yet") { + t.Fatalf("a read of a deleted bucket answered %v, want that its bucket is not on the bus", err) + } + // And made afresh: read again from the new bucket. + again, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: bucket, History: 1}) + if err != nil { + t.Fatal(err) + } + if _, err := again.Put("a", []byte(`{"v":2}`)); err != nil { + t.Fatal(err) + } + e, err := conn.StateGet("gitea", "mesh-delivery-assigner.pins", "a") + if err != nil || e == nil || string(e.Value) != `{"v":2}` { + t.Fatalf("a read of the bucket made afresh: %+v, %v", e, err) + } + read() + if err := watcher.Flush(); err != nil { + t.Fatal(err) + } + if got := infos.Load(); got != 3 { + t.Fatalf("the bucket's stream info was asked %d times in all, want 3: once, once more when it was gone, once when made afresh", got) + } +} diff --git a/node-tools/internal/bus/state_bind_test.go b/node-tools/internal/bus/state_bind_test.go new file mode 100644 index 0000000..db855fd --- /dev/null +++ b/node-tools/internal/bus/state_bind_test.go @@ -0,0 +1,141 @@ +package bus + +import ( + "context" + "encoding/json" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/nats-io/nats.go/jetstream" +) + +// boundKV is a bucket's binding in a test: it answers every get with one value, or, once gone, as the bus answers a +// binding whose bucket was deleted. +type boundKV struct { + jetstream.KeyValue + gone atomic.Bool + puts atomic.Int32 +} + +type boundEntry struct { + jetstream.KeyValueEntry + key string +} + +func (e boundEntry) Key() string { return e.key } +func (e boundEntry) Value() []byte { return []byte(`{"v":1}`) } +func (e boundEntry) Revision() uint64 { return 1 } +func (boundEntry) Created() time.Time { return time.Time{} } +func (*boundKV) Bucket() string { return "gitea_pins" } +func (kv *boundKV) Get(_ context.Context, key string) (jetstream.KeyValueEntry, error) { + if kv.gone.Load() { + return nil, jetstream.ErrStreamNotFound + } + return boundEntry{key: key}, nil +} +func (kv *boundKV) Put(context.Context, string, []byte) (uint64, error) { + if kv.gone.Load() { + return 0, jetstream.ErrNoStreamResponse + } + return uint64(kv.puts.Add(1)), nil +} + +// **A state is bound once for the process, and bound again only when the bus says its bucket is gone** (design 47 H +// lead answers H-A-251): binding is a stream-info request on the bus, and asked before every get it made each read two +// answers on the bus. +func TestAStateIsBoundOnceAndBoundAgainWhenItsBucketIsGone(t *testing.T) { + setup := func() (*Conn, *atomic.Int32, *[]*boundKV) { + var binds atomic.Int32 + var mu sync.Mutex + made := []*boundKV{} + c := &Conn{node: "anchor", issued: map[string]*Membership{"gitea": {Module: "gitea", + State: []StateIssued{{Name: "pins", Bucket: "gitea_pins", Writes: true}}}}} + c.bind = func(_ context.Context, bucket string) (jetstream.KeyValue, error) { + binds.Add(1) + if bucket != "gitea_pins" { + t.Errorf("bound %s, want gitea_pins", bucket) + } + kv := &boundKV{} + mu.Lock() + made = append(made, kv) + mu.Unlock() + return kv, nil + } + return c, &binds, &made + } + + t.Run("two reads, one bind", func(t *testing.T) { + c, binds, _ := setup() + for range 2 { + if e, err := c.StateGet("gitea", "pins", "a"); err != nil || e == nil || e.Key != "a" { + t.Fatalf("read: %+v, %v", e, err) + } + } + if _, err := c.StatePut("gitea", "pins", "a", json.RawMessage(`{"v":2}`)); err != nil { + t.Fatal(err) + } + if got := binds.Load(); got != 1 { + t.Fatalf("two reads and a write bound the bucket %d times, want once", got) + } + }) + + t.Run("many readers at once, one bind", func(t *testing.T) { + c, binds, _ := setup() + var wg sync.WaitGroup + for range 16 { + wg.Go(func() { + if _, err := c.StateGet("gitea", "pins", "a"); err != nil { + t.Error(err) + } + }) + } + wg.Wait() + if got := binds.Load(); got != 1 { + t.Fatalf("16 readers bound the bucket %d times, want once", got) + } + }) + + t.Run("a bucket gone under the runner is bound again once", func(t *testing.T) { + c, binds, made := setup() + if _, err := c.StateGet("gitea", "pins", "a"); err != nil { + t.Fatal(err) + } + (*made)[0].gone.Store(true) + if e, err := c.StateGet("gitea", "pins", "a"); err != nil || e == nil { + t.Fatalf("the read after the bucket was made again: %+v, %v", e, err) + } + if got := binds.Load(); got != 2 { + t.Fatalf("the bucket was bound %d times, want twice (once more after it was gone)", got) + } + if _, err := c.StateGet("gitea", "pins", "a"); err != nil { + t.Fatal(err) + } + (*made)[1].gone.Store(true) + if _, err := c.StatePut("gitea", "pins", "a", json.RawMessage(`{"v":2}`)); err != nil { + t.Fatal(err) + } + if got, puts := binds.Load(), (*made)[2].puts.Load(); got != 3 || puts != 1 { + t.Fatalf("a write to a gone bucket bound it %d times in all (want 3) and wrote %d times on the new binding (want 1)", got, puts) + } + }) + + t.Run("a bucket still gone after the bind again is said", func(t *testing.T) { + c, binds, made := setup() + if _, err := c.StateGet("gitea", "pins", "a"); err != nil { + t.Fatal(err) + } + (*made)[0].gone.Store(true) + c.bind = func(context.Context, string) (jetstream.KeyValue, error) { + binds.Add(1) + return nil, jetstream.ErrBucketNotFound + } + if _, err := c.StateGet("gitea", "pins", "a"); err == nil { + t.Fatal("a read of a bucket no longer on the bus answered no error") + } + if got := binds.Load(); got != 2 { + t.Fatalf("the bucket was bound %d times, want twice: the bind again is once", got) + } + }) +} -- 2.54.0