diff --git a/node-tools/internal/bus/bus.go b/node-tools/internal/bus/bus.go index 726dc0c..c13a122 100644 --- a/node-tools/internal/bus/bus.go +++ b/node-tools/internal/bus/bus.go @@ -203,6 +203,13 @@ type Conn struct { waits map[string]map[chan string]bool // inboxSaid is when a refused answer publish was last said, by subject (answerRefused). inboxSaid map[string]time.Time + // bound is each state's binding, kept for the process and bound again only when the bus says its bucket is gone + // (state.go, design 47 H lead answers H-A-251): one Conn is one account's connection, so a binding is per account + // and bucket. + bindMu sync.Mutex + bound map[string]*binding + // bind binds one bucket: jetstream's KeyValue (a stream-info request on the bus) unless a test replaces it. + bind func(ctx context.Context, bucket string) (jetstream.KeyValue, error) } // answered is one subject the runtime answers on: what to subscribe again with, and how often it was diff --git a/node-tools/internal/bus/seat.go b/node-tools/internal/bus/seat.go index 1ef7fc1..f703d16 100644 --- a/node-tools/internal/bus/seat.go +++ b/node-tools/internal/bus/seat.go @@ -427,11 +427,7 @@ func (c *Conn) stateWrite(module, name, key string, value json.RawMessage, } ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) defer cancel() - kv, err := c.bucket(ctx, s) - if err != nil { - return 0, err - } - rev, err := write(ctx, kv) + rev, err := inBucket(c, ctx, s, func(kv jetstream.KeyValue) (uint64, error) { return write(ctx, kv) }) if errors.Is(err, jetstream.ErrKeyExists) || isWrongSequence(err) { return 0, changed{name + "." + key} } diff --git a/node-tools/internal/bus/state.go b/node-tools/internal/bus/state.go index 1e482f9..a68e9ce 100644 --- a/node-tools/internal/bus/state.go +++ b/node-tools/internal/bus/state.go @@ -15,6 +15,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/membership" "git.novox.be/novox/mesh-sdk/go/pernode" + "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" ) @@ -92,20 +93,107 @@ func (c *Conn) issuedState(module, name string) (StateIssued, error) { "issued: %s (novox/hq ADR 0201, ADR 0595 ยง1)", module, name, issued) } +// binding is one bucket's binding, or the bind in flight for it: the first asker binds, an asker meanwhile waits for +// that bind (done) instead of asking the bus again. +type binding struct { + done chan struct{} + kv jetstream.KeyValue + err error +} + +// bucket is the state's binding, bound once for the process (design 47 H lead answers H-A-251): binding a bucket is a +// stream-info request on the bus, and asked before every get it made each read two answers. A bind that fails is not +// kept, so the next ask binds again. func (c *Conn) bucket(ctx context.Context, s StateIssued) (jetstream.KeyValue, error) { + c.bindMu.Lock() + b := c.bound[s.Bucket] + if b == nil { + if c.bound == nil { + c.bound = map[string]*binding{} + } + b = &binding{done: make(chan struct{})} + c.bound[s.Bucket] = b + c.bindMu.Unlock() + b.kv, b.err = c.bindBucket(ctx, s.Bucket) + if b.err != nil { + c.bindMu.Lock() + if c.bound[s.Bucket] == b { + delete(c.bound, s.Bucket) + } + c.bindMu.Unlock() + } + close(b.done) + } else { + c.bindMu.Unlock() + select { + case <-b.done: + case <-ctx.Done(): + return nil, ctx.Err() + } + } + if b.err != nil { + if errors.Is(b.err, jetstream.ErrBucketNotFound) { + return nil, fmt.Errorf("the state %q is issued and its bucket is not on the bus yet: the mesh "+ + "creates it from the catalogue on its next raise", s.Name) + } + return nil, b.err + } + return b.kv, nil +} + +func (c *Conn) bindBucket(ctx context.Context, bucket string) (jetstream.KeyValue, error) { + if c.bind != nil { + return c.bind(ctx, bucket) + } js, err := jetstream.New(c.nc) if err != nil { return nil, err } - kv, err := js.KeyValue(ctx, s.Bucket) - if err != nil { - if errors.Is(err, jetstream.ErrBucketNotFound) { - return nil, fmt.Errorf("the state %q is issued and its bucket is not on the bus yet: the mesh "+ - "creates it from the catalogue on its next raise", s.Name) - } - return nil, err + return js.KeyValue(ctx, bucket) +} + +// unbind drops a bucket's binding the bus said is gone, when it is still the one kept, so the next ask binds again. +func (c *Conn) unbind(bucket string, kv jetstream.KeyValue) { + c.bindMu.Lock() + defer c.bindMu.Unlock() + b := c.bound[bucket] + if b == nil { + return } - return kv, nil + select { + case <-b.done: // bound: its kv is written + default: + return // bound again already, in flight + } + if b.kv == kv { + delete(c.bound, bucket) + } +} + +// bucketGone is the bus saying a kept binding's bucket is not there (deleted, or deleted and created again): no stream of +// the name, or nobody answers on its subjects. +func bucketGone(err error) bool { + return errors.Is(err, jetstream.ErrStreamNotFound) || errors.Is(err, jetstream.ErrBucketNotFound) || + errors.Is(err, jetstream.ErrNoStreamResponse) || errors.Is(err, nats.ErrNoResponders) +} + +// inBucket does one act on the state's kept binding; when the bus says its bucket is gone, the binding is dropped, the +// bucket bound again once and the act done once more on the new binding (H-A-251). +func inBucket[T any](c *Conn, ctx context.Context, s StateIssued, do func(jetstream.KeyValue) (T, error)) (T, error) { + var none T + kv, err := c.bucket(ctx, s) + if err != nil { + return none, err + } + got, err := do(kv) + if !bucketGone(err) { + return got, err + } + c.unbind(s.Bucket, kv) + if kv, err = c.bucket(ctx, s); err != nil { + return none, err + } + return do(kv) } func checkKey(key string) error { @@ -130,11 +218,7 @@ func (c *Conn) StateGet(module, name, key string) (*StateEntry, error) { } ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) defer cancel() - kv, err := c.bucket(ctx, s) - if err != nil { - return nil, err - } - e, err := kv.Get(ctx, key) + e, err := inBucket(c, ctx, s, func(kv jetstream.KeyValue) (jetstream.KeyValueEntry, error) { return kv.Get(ctx, key) }) if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, nil } @@ -210,11 +294,10 @@ func (c *Conn) StatePage(module, name, after string, maxBytes int, budget time.D maps.Copy(current, got) } } else { - kv, err := c.bucket(ctx, s) - if err != nil { - return StatePage{}, err - } - if current, err = currentOf(ctx, kv); err != nil { + var err error + if current, err = inBucket(c, ctx, s, func(kv jetstream.KeyValue) (map[string]*StateEntry, error) { + return currentOf(ctx, kv) + }); err != nil { return StatePage{}, err } } @@ -273,11 +356,9 @@ func (c *Conn) StateHistory(module, name, key string) ([]StateKept, error) { } ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) defer cancel() - kv, err := c.bucket(ctx, s) - if err != nil { - return nil, err - } - entries, err := kv.History(ctx, key) + entries, err := inBucket(c, ctx, s, func(kv jetstream.KeyValue) ([]jetstream.KeyValueEntry, error) { + return kv.History(ctx, key) + }) kept := []StateKept{} if errors.Is(err, jetstream.ErrKeyNotFound) { return kept, nil @@ -326,11 +407,7 @@ func (c *Conn) StateInfo(module, name string) (*StateInfo, error) { } ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) defer cancel() - kv, err := c.bucket(ctx, s) - if err != nil { - return nil, err - } - status, err := kv.Status(ctx) + status, err := inBucket(c, ctx, s, func(kv jetstream.KeyValue) (jetstream.KeyValueStatus, error) { return kv.Status(ctx) }) if err != nil { return nil, err } @@ -363,11 +440,7 @@ func (c *Conn) StatePut(module, name, key string, value json.RawMessage) (uint64 } ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) defer cancel() - kv, err := c.bucket(ctx, s) - if err != nil { - return 0, err - } - return kv.Put(ctx, key, value) + return inBucket(c, ctx, s, func(kv jetstream.KeyValue) (uint64, error) { return kv.Put(ctx, key, value) }) } // MinKeyTTL is the shortest time to live a key may be given: the bus counts a key's age in whole seconds (mesh-sdk @@ -437,11 +510,8 @@ func (c *Conn) StateDelete(module, name, key string) error { } ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) defer cancel() - kv, err := c.bucket(ctx, s) - if err != nil { - return err - } - return kv.Delete(ctx, key) + _, err = inBucket(c, ctx, s, func(kv jetstream.KeyValue) (struct{}, error) { return struct{}{}, kv.Delete(ctx, key) }) + return err } // StateKeys is every key with a current value, sorted. @@ -455,11 +525,7 @@ func (c *Conn) StateKeys(module, name string) ([]string, error) { } ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) defer cancel() - kv, err := c.bucket(ctx, s) - if err != nil { - return nil, err - } - return keysOf(ctx, kv) + return inBucket(c, ctx, s, func(kv jetstream.KeyValue) ([]string, error) { return keysOf(ctx, kv) }) } // keysOf is every key with a current value in one bucket, sorted. @@ -529,12 +595,16 @@ func (c *Conn) StateWatch(module, name, pattern string, deliver func(StateChange return c.perNodeWatch(module, s, pattern, deliver) } ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) - kv, err := c.bucket(ctx, s) - cancel() + defer cancel() + type watching struct{ stop func() } + w, err := inBucket(c, ctx, s, func(kv jetstream.KeyValue) (watching, error) { + stop, err := c.watchBucket(module, name, kv, filter, deliver) + return watching{stop}, err + }) if err != nil { return nil, err } - return c.watchBucket(module, name, kv, filter, deliver) + return w.stop, nil } // watchBucket is StateWatch over one bucket: the current values, then every change. diff --git a/node-tools/internal/bus/state_bind_integration_test.go b/node-tools/internal/bus/state_bind_integration_test.go new file mode 100644 index 0000000..22e4a20 --- /dev/null +++ b/node-tools/internal/bus/state_bind_integration_test.go @@ -0,0 +1,81 @@ +//go:build integration + +package bus_test + +import ( + "strings" + "sync/atomic" + "testing" + + "github.com/nats-io/nats.go" + + mt "github.com/novox/mesh-tools/node-tools/internal/meshtest" +) + +// **On the bus, a read of a state is one answer: its bucket is bound once, and bound again when it was made afresh or +// is gone** (design 47 H lead answers H-A-251). The runtime asked the bucket's stream info before every get, so each +// read was two answers; a third of a night's bus flood came this way. +func TestAStateReadOnTheBusBindsItsBucketOnce(t *testing.T) { + conn, kv := owedWatcher(t) + const bucket = "mesh-delivery-assigner_pins" + if _, err := kv.Put("a", []byte(`{"v":1}`)); err != nil { + t.Fatal(err) + } + watcher, err := nats.Connect(mt.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(watcher.Close) + var infos atomic.Int32 + if _, err := watcher.Subscribe("$JS.API.STREAM.INFO.KV_"+bucket, func(*nats.Msg) { infos.Add(1) }); err != nil { + t.Fatal(err) + } + if err := watcher.Flush(); err != nil { + t.Fatal(err) + } + read := func() { + t.Helper() + e, err := conn.StateGet("gitea", "mesh-delivery-assigner.pins", "a") + if err != nil || e == nil { + t.Fatalf("read: %+v, %v", e, err) + } + } + read() + read() + read() + if err := watcher.Flush(); err != nil { + t.Fatal(err) + } + if got := infos.Load(); got != 1 { + t.Fatalf("three reads asked the bucket's stream info %d times, want once", got) + } + + // The bucket deleted under the runtime: the read binds again, once, and says the bucket is not on the bus. + js, _ := watcher.JetStream() + if err := js.DeleteKeyValue(bucket); err != nil { + t.Fatal(err) + } + _, err = conn.StateGet("gitea", "mesh-delivery-assigner.pins", "a") + if err == nil || !strings.Contains(err.Error(), "not on the bus yet") { + t.Fatalf("a read of a deleted bucket answered %v, want that its bucket is not on the bus", err) + } + // And made afresh: read again from the new bucket. + again, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: bucket, History: 1}) + if err != nil { + t.Fatal(err) + } + if _, err := again.Put("a", []byte(`{"v":2}`)); err != nil { + t.Fatal(err) + } + e, err := conn.StateGet("gitea", "mesh-delivery-assigner.pins", "a") + if err != nil || e == nil || string(e.Value) != `{"v":2}` { + t.Fatalf("a read of the bucket made afresh: %+v, %v", e, err) + } + read() + if err := watcher.Flush(); err != nil { + t.Fatal(err) + } + if got := infos.Load(); got != 3 { + t.Fatalf("the bucket's stream info was asked %d times in all, want 3: once, once more when it was gone, once when made afresh", got) + } +} diff --git a/node-tools/internal/bus/state_bind_test.go b/node-tools/internal/bus/state_bind_test.go new file mode 100644 index 0000000..db855fd --- /dev/null +++ b/node-tools/internal/bus/state_bind_test.go @@ -0,0 +1,141 @@ +package bus + +import ( + "context" + "encoding/json" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/nats-io/nats.go/jetstream" +) + +// boundKV is a bucket's binding in a test: it answers every get with one value, or, once gone, as the bus answers a +// binding whose bucket was deleted. +type boundKV struct { + jetstream.KeyValue + gone atomic.Bool + puts atomic.Int32 +} + +type boundEntry struct { + jetstream.KeyValueEntry + key string +} + +func (e boundEntry) Key() string { return e.key } +func (e boundEntry) Value() []byte { return []byte(`{"v":1}`) } +func (e boundEntry) Revision() uint64 { return 1 } +func (boundEntry) Created() time.Time { return time.Time{} } +func (*boundKV) Bucket() string { return "gitea_pins" } +func (kv *boundKV) Get(_ context.Context, key string) (jetstream.KeyValueEntry, error) { + if kv.gone.Load() { + return nil, jetstream.ErrStreamNotFound + } + return boundEntry{key: key}, nil +} +func (kv *boundKV) Put(context.Context, string, []byte) (uint64, error) { + if kv.gone.Load() { + return 0, jetstream.ErrNoStreamResponse + } + return uint64(kv.puts.Add(1)), nil +} + +// **A state is bound once for the process, and bound again only when the bus says its bucket is gone** (design 47 H +// lead answers H-A-251): binding is a stream-info request on the bus, and asked before every get it made each read two +// answers on the bus. +func TestAStateIsBoundOnceAndBoundAgainWhenItsBucketIsGone(t *testing.T) { + setup := func() (*Conn, *atomic.Int32, *[]*boundKV) { + var binds atomic.Int32 + var mu sync.Mutex + made := []*boundKV{} + c := &Conn{node: "anchor", issued: map[string]*Membership{"gitea": {Module: "gitea", + State: []StateIssued{{Name: "pins", Bucket: "gitea_pins", Writes: true}}}}} + c.bind = func(_ context.Context, bucket string) (jetstream.KeyValue, error) { + binds.Add(1) + if bucket != "gitea_pins" { + t.Errorf("bound %s, want gitea_pins", bucket) + } + kv := &boundKV{} + mu.Lock() + made = append(made, kv) + mu.Unlock() + return kv, nil + } + return c, &binds, &made + } + + t.Run("two reads, one bind", func(t *testing.T) { + c, binds, _ := setup() + for range 2 { + if e, err := c.StateGet("gitea", "pins", "a"); err != nil || e == nil || e.Key != "a" { + t.Fatalf("read: %+v, %v", e, err) + } + } + if _, err := c.StatePut("gitea", "pins", "a", json.RawMessage(`{"v":2}`)); err != nil { + t.Fatal(err) + } + if got := binds.Load(); got != 1 { + t.Fatalf("two reads and a write bound the bucket %d times, want once", got) + } + }) + + t.Run("many readers at once, one bind", func(t *testing.T) { + c, binds, _ := setup() + var wg sync.WaitGroup + for range 16 { + wg.Go(func() { + if _, err := c.StateGet("gitea", "pins", "a"); err != nil { + t.Error(err) + } + }) + } + wg.Wait() + if got := binds.Load(); got != 1 { + t.Fatalf("16 readers bound the bucket %d times, want once", got) + } + }) + + t.Run("a bucket gone under the runner is bound again once", func(t *testing.T) { + c, binds, made := setup() + if _, err := c.StateGet("gitea", "pins", "a"); err != nil { + t.Fatal(err) + } + (*made)[0].gone.Store(true) + if e, err := c.StateGet("gitea", "pins", "a"); err != nil || e == nil { + t.Fatalf("the read after the bucket was made again: %+v, %v", e, err) + } + if got := binds.Load(); got != 2 { + t.Fatalf("the bucket was bound %d times, want twice (once more after it was gone)", got) + } + if _, err := c.StateGet("gitea", "pins", "a"); err != nil { + t.Fatal(err) + } + (*made)[1].gone.Store(true) + if _, err := c.StatePut("gitea", "pins", "a", json.RawMessage(`{"v":2}`)); err != nil { + t.Fatal(err) + } + if got, puts := binds.Load(), (*made)[2].puts.Load(); got != 3 || puts != 1 { + t.Fatalf("a write to a gone bucket bound it %d times in all (want 3) and wrote %d times on the new binding (want 1)", got, puts) + } + }) + + t.Run("a bucket still gone after the bind again is said", func(t *testing.T) { + c, binds, made := setup() + if _, err := c.StateGet("gitea", "pins", "a"); err != nil { + t.Fatal(err) + } + (*made)[0].gone.Store(true) + c.bind = func(context.Context, string) (jetstream.KeyValue, error) { + binds.Add(1) + return nil, jetstream.ErrBucketNotFound + } + if _, err := c.StateGet("gitea", "pins", "a"); err == nil { + t.Fatal("a read of a bucket no longer on the bus answered no error") + } + if got := binds.Load(); got != 2 { + t.Fatalf("the bucket was bound %d times, want twice: the bind again is once", got) + } + }) +}