Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ee143c9f91 | ||
|
|
05d8f6b762 | ||
|
|
fe707779e3 | ||
|
|
3eef0254df | ||
|
|
07f5aa11c3 | ||
|
|
876adfbf05 | ||
|
|
c952386737 |
@@ -196,19 +196,28 @@ func (c *Conn) ObjectsList(module string, r buscalls.ObjectsRequest) (buscalls.O
|
|||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ObjectsInfo is how full the module's object store is: its objects, its bytes and its cap.
|
// ObjectsInfo is how full the module's object store is, from the figures the bus keeps for its stream: its bytes, its
|
||||||
|
// messages' live subjects as the count, and its cap. It never lists the store: a store's descriptions, the leftovers
|
||||||
|
// of deleted objects among them, may be many thousands, and a fill read per chunk of a transfer read them all each
|
||||||
|
// time (design 47 H lead answers H-A-227).
|
||||||
func (c *Conn) ObjectsInfo(module string, r buscalls.ObjectsRequest) (buscalls.Fill, error) {
|
func (c *Conn) ObjectsInfo(module string, r buscalls.ObjectsRequest) (buscalls.Fill, error) {
|
||||||
listed, err := c.ObjectsList(module, r)
|
b, err := c.objectBucketOf(module, r.Objects)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return buscalls.Fill{}, err
|
return buscalls.Fill{}, err
|
||||||
}
|
}
|
||||||
b, _ := c.objectBucketOf(module, r.Objects)
|
info, err := c.streamInfo("OBJ_" + b.ObjectStore)
|
||||||
fill, err := c.streamFill("OBJ_" + b.ObjectStore)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return buscalls.Fill{}, err
|
return buscalls.Fill{}, err
|
||||||
}
|
}
|
||||||
fill.Count = uint64(len(listed.Objects))
|
return objectsFillOf(info), nil
|
||||||
return fill.Fill, nil
|
}
|
||||||
|
|
||||||
|
// objectsFillOf is an object store's fill from its stream's info alone: the bytes the stream keeps, its live subjects
|
||||||
|
// as the count (H-A-227: the stream's live subjects, or none) and its cap.
|
||||||
|
func objectsFillOf(info *jetstream.StreamInfo) buscalls.Fill {
|
||||||
|
fill := streamFillOf(info).Fill
|
||||||
|
fill.Count = uint64(info.State.NumSubjects)
|
||||||
|
return fill
|
||||||
}
|
}
|
||||||
|
|
||||||
// ObjectsWatch hands deliver every object of the module's object store as it is, then each stored or deleted.
|
// ObjectsWatch hands deliver every object of the module's object store as it is, then each stored or deleted.
|
||||||
@@ -261,29 +270,39 @@ func (c *Conn) ObjectsWatch(module string, r buscalls.ObjectsWatchRequest, deliv
|
|||||||
|
|
||||||
// streamFill is a stream's fill and its first and last sequence, by its name on the bus.
|
// streamFill is a stream's fill and its first and last sequence, by its name on the bus.
|
||||||
func (c *Conn) streamFill(name string) (buscalls.StreamFilled, error) {
|
func (c *Conn) streamFill(name string) (buscalls.StreamFilled, error) {
|
||||||
|
info, err := c.streamInfo(name)
|
||||||
|
if err != nil {
|
||||||
|
return buscalls.StreamFilled{}, err
|
||||||
|
}
|
||||||
|
return streamFillOf(info), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// streamInfo is a stream's info as the bus keeps it, by its name on the bus: one request, whatever the stream holds.
|
||||||
|
func (c *Conn) streamInfo(name string) (*jetstream.StreamInfo, error) {
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
js, err := c.JetStream()
|
js, err := c.JetStream()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return buscalls.StreamFilled{}, err
|
return nil, err
|
||||||
}
|
}
|
||||||
s, err := js.Stream(ctx, name)
|
s, err := js.Stream(ctx, name)
|
||||||
if errors.Is(err, jetstream.ErrStreamNotFound) {
|
if errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||||
return buscalls.StreamFilled{}, fmt.Errorf("the stream %s is bound and not on the bus", name)
|
return nil, fmt.Errorf("the stream %s is bound and not on the bus", name)
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return buscalls.StreamFilled{}, err
|
return nil, err
|
||||||
}
|
|
||||||
info, err := s.Info(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return buscalls.StreamFilled{}, err
|
|
||||||
}
|
}
|
||||||
|
return s.Info(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
// streamFillOf is a stream's fill and its first and last sequence from its info.
|
||||||
|
func streamFillOf(info *jetstream.StreamInfo) buscalls.StreamFilled {
|
||||||
max := uint64(0)
|
max := uint64(0)
|
||||||
if info.Config.MaxBytes > 0 {
|
if info.Config.MaxBytes > 0 {
|
||||||
max = uint64(info.Config.MaxBytes)
|
max = uint64(info.Config.MaxBytes)
|
||||||
}
|
}
|
||||||
return buscalls.StreamFilled{Fill: buscalls.Fill{Count: info.State.Msgs, Bytes: info.State.Bytes, MaxBytes: max},
|
return buscalls.StreamFilled{Fill: buscalls.Fill{Count: info.State.Msgs, Bytes: info.State.Bytes, MaxBytes: max},
|
||||||
First: info.State.FirstSeq, Last: info.State.LastSeq}, nil
|
First: info.State.FirstSeq, Last: info.State.LastSeq}
|
||||||
}
|
}
|
||||||
|
|
||||||
// streamBinding is the module's binding of a stream by its local name, a work queue among them.
|
// streamBinding is the module's binding of a stream by its local name, a work queue among them.
|
||||||
|
|||||||
@@ -95,6 +95,11 @@ func TestABundleReachesItsOwnBoundObjectsAndStreamsOnly(t *testing.T) {
|
|||||||
if listed, err := conn.ObjectsList("m", buscalls.ObjectsRequest{Objects: "files"}); err != nil || len(listed.Objects) != 1 {
|
if listed, err := conn.ObjectsList("m", buscalls.ObjectsRequest{Objects: "files"}); err != nil || len(listed.Objects) != 1 {
|
||||||
t.Fatalf("listed %+v, %v", listed, err)
|
t.Fatalf("listed %+v, %v", listed, err)
|
||||||
}
|
}
|
||||||
|
// H-A-227: the fill is the stream's own figures, its live subjects as the count (a's chunk and description, big's
|
||||||
|
// description left by its delete), never the count of its listed objects.
|
||||||
|
if fill, err := conn.ObjectsInfo("m", buscalls.ObjectsRequest{Objects: "files"}); err != nil || fill.Count != 3 || fill.Bytes == 0 {
|
||||||
|
t.Fatalf("objects fill %+v, %v", fill, err)
|
||||||
|
}
|
||||||
// A-380: an old `objects` entry is reached by the store grants creates for it, `<module>_<name>`, with no binding of that name.
|
// A-380: an old `objects` entry is reached by the store grants creates for it, `<module>_<name>`, with no binding of that name.
|
||||||
if _, err := conn.ObjectsPut("m", buscalls.ObjectsPutRequest{Objects: "old", Name: "r", Data: []byte("old section")}); err != nil {
|
if _, err := conn.ObjectsPut("m", buscalls.ObjectsPutRequest{Objects: "old", Name: "r", Data: []byte("old section")}); err != nil {
|
||||||
t.Fatalf("put into the old objects section: %v", err)
|
t.Fatalf("put into the old objects section: %v", err)
|
||||||
|
|||||||
@@ -203,6 +203,13 @@ type Conn struct {
|
|||||||
waits map[string]map[chan string]bool
|
waits map[string]map[chan string]bool
|
||||||
// inboxSaid is when a refused answer publish was last said, by subject (answerRefused).
|
// inboxSaid is when a refused answer publish was last said, by subject (answerRefused).
|
||||||
inboxSaid map[string]time.Time
|
inboxSaid map[string]time.Time
|
||||||
|
// bound is each state's binding, kept for the process and bound again only when the bus says its bucket is gone
|
||||||
|
// (state.go, design 47 H lead answers H-A-251): one Conn is one account's connection, so a binding is per account
|
||||||
|
// and bucket.
|
||||||
|
bindMu sync.Mutex
|
||||||
|
bound map[string]*binding
|
||||||
|
// bind binds one bucket: jetstream's KeyValue (a stream-info request on the bus) unless a test replaces it.
|
||||||
|
bind func(ctx context.Context, bucket string) (jetstream.KeyValue, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
// answered is one subject the runtime answers on: what to subscribe again with, and how often it was
|
// answered is one subject the runtime answers on: what to subscribe again with, and how often it was
|
||||||
|
|||||||
@@ -77,6 +77,8 @@ func TestAConsumerSaysItIsBoundWhileIdleAndOnceWhenTheBusEndedIt(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}))
|
}))
|
||||||
mesh.Consumer(t, "anchor", "quiet", []string{"mesh.mod.alpha.event.>"}, time.Second)
|
mesh.Consumer(t, "anchor", "quiet", []string{"mesh.mod.alpha.event.>"}, time.Second)
|
||||||
|
// The unasked consume below pulls a durable of its own: a second loop on quiet's is refused (H-A-196, H-A-197).
|
||||||
|
mesh.Consumer(t, "anchor", "unasked", []string{"mesh.mod.alpha.event.>"}, time.Second)
|
||||||
c, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: "node-tools", Node: "anchor"})
|
c, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: "node-tools", Node: "anchor"})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -97,7 +99,7 @@ func TestAConsumerSaysItIsBoundWhileIdleAndOnceWhenTheBusEndedIt(t *testing.T) {
|
|||||||
t.Cleanup(stop)
|
t.Cleanup(stop)
|
||||||
// A consume nobody asked a heartbeat of says nothing and asks the bus nothing, however many intervals pass.
|
// A consume nobody asked a heartbeat of says nothing and asks the bus nothing, however many intervals pass.
|
||||||
var unaskedBeats atomic.Int32
|
var unaskedBeats atomic.Int32
|
||||||
unasked, err := c.ConsumeAsBeatingWhen("quiet", func(bus.Envelope) error { return nil },
|
unasked, err := c.ConsumeAsBeatingWhen("unasked", func(bus.Envelope) error { return nil },
|
||||||
func(bus.Beat) { unaskedBeats.Add(1) }, func() bool { return false })
|
func(bus.Beat) { unaskedBeats.Add(1) }, func() bool { return false })
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
|
|||||||
@@ -427,11 +427,7 @@ func (c *Conn) stateWrite(module, name, key string, value json.RawMessage,
|
|||||||
}
|
}
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
kv, err := c.bucket(ctx, s)
|
rev, err := inBucket(c, ctx, s, func(kv jetstream.KeyValue) (uint64, error) { return write(ctx, kv) })
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
rev, err := write(ctx, kv)
|
|
||||||
if errors.Is(err, jetstream.ErrKeyExists) || isWrongSequence(err) {
|
if errors.Is(err, jetstream.ErrKeyExists) || isWrongSequence(err) {
|
||||||
return 0, changed{name + "." + key}
|
return 0, changed{name + "." + key}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ import (
|
|||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/membership"
|
"git.novox.be/novox/mesh-sdk/go/membership"
|
||||||
"git.novox.be/novox/mesh-sdk/go/pernode"
|
"git.novox.be/novox/mesh-sdk/go/pernode"
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
"github.com/nats-io/nats.go/jetstream"
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -92,20 +93,107 @@ func (c *Conn) issuedState(module, name string) (StateIssued, error) {
|
|||||||
"issued: %s (novox/hq ADR 0201, ADR 0595 §1)", module, name, issued)
|
"issued: %s (novox/hq ADR 0201, ADR 0595 §1)", module, name, issued)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// binding is one bucket's binding, or the bind in flight for it: the first asker binds, an asker meanwhile waits for
|
||||||
|
// that bind (done) instead of asking the bus again.
|
||||||
|
type binding struct {
|
||||||
|
done chan struct{}
|
||||||
|
kv jetstream.KeyValue
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
// bucket is the state's binding, bound once for the process (design 47 H lead answers H-A-251): binding a bucket is a
|
||||||
|
// stream-info request on the bus, and asked before every get it made each read two answers. A bind that fails is not
|
||||||
|
// kept, so the next ask binds again.
|
||||||
func (c *Conn) bucket(ctx context.Context, s StateIssued) (jetstream.KeyValue, error) {
|
func (c *Conn) bucket(ctx context.Context, s StateIssued) (jetstream.KeyValue, error) {
|
||||||
|
c.bindMu.Lock()
|
||||||
|
b := c.bound[s.Bucket]
|
||||||
|
if b == nil {
|
||||||
|
if c.bound == nil {
|
||||||
|
c.bound = map[string]*binding{}
|
||||||
|
}
|
||||||
|
b = &binding{done: make(chan struct{})}
|
||||||
|
c.bound[s.Bucket] = b
|
||||||
|
c.bindMu.Unlock()
|
||||||
|
b.kv, b.err = c.bindBucket(ctx, s.Bucket)
|
||||||
|
if b.err != nil {
|
||||||
|
c.bindMu.Lock()
|
||||||
|
if c.bound[s.Bucket] == b {
|
||||||
|
delete(c.bound, s.Bucket)
|
||||||
|
}
|
||||||
|
c.bindMu.Unlock()
|
||||||
|
}
|
||||||
|
close(b.done)
|
||||||
|
} else {
|
||||||
|
c.bindMu.Unlock()
|
||||||
|
select {
|
||||||
|
case <-b.done:
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, ctx.Err()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if b.err != nil {
|
||||||
|
if errors.Is(b.err, jetstream.ErrBucketNotFound) {
|
||||||
|
return nil, fmt.Errorf("the state %q is issued and its bucket is not on the bus yet: the mesh "+
|
||||||
|
"creates it from the catalogue on its next raise", s.Name)
|
||||||
|
}
|
||||||
|
return nil, b.err
|
||||||
|
}
|
||||||
|
return b.kv, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Conn) bindBucket(ctx context.Context, bucket string) (jetstream.KeyValue, error) {
|
||||||
|
if c.bind != nil {
|
||||||
|
return c.bind(ctx, bucket)
|
||||||
|
}
|
||||||
js, err := jetstream.New(c.nc)
|
js, err := jetstream.New(c.nc)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
kv, err := js.KeyValue(ctx, s.Bucket)
|
return js.KeyValue(ctx, bucket)
|
||||||
if err != nil {
|
}
|
||||||
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
|
||||||
return nil, fmt.Errorf("the state %q is issued and its bucket is not on the bus yet: the mesh "+
|
// unbind drops a bucket's binding the bus said is gone, when it is still the one kept, so the next ask binds again.
|
||||||
"creates it from the catalogue on its next raise", s.Name)
|
func (c *Conn) unbind(bucket string, kv jetstream.KeyValue) {
|
||||||
}
|
c.bindMu.Lock()
|
||||||
return nil, err
|
defer c.bindMu.Unlock()
|
||||||
|
b := c.bound[bucket]
|
||||||
|
if b == nil {
|
||||||
|
return
|
||||||
}
|
}
|
||||||
return kv, nil
|
select {
|
||||||
|
case <-b.done: // bound: its kv is written
|
||||||
|
default:
|
||||||
|
return // bound again already, in flight
|
||||||
|
}
|
||||||
|
if b.kv == kv {
|
||||||
|
delete(c.bound, bucket)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// bucketGone is the bus saying a kept binding's bucket is not there (deleted, or deleted and created again): no stream of
|
||||||
|
// the name, or nobody answers on its subjects.
|
||||||
|
func bucketGone(err error) bool {
|
||||||
|
return errors.Is(err, jetstream.ErrStreamNotFound) || errors.Is(err, jetstream.ErrBucketNotFound) ||
|
||||||
|
errors.Is(err, jetstream.ErrNoStreamResponse) || errors.Is(err, nats.ErrNoResponders)
|
||||||
|
}
|
||||||
|
|
||||||
|
// inBucket does one act on the state's kept binding; when the bus says its bucket is gone, the binding is dropped, the
|
||||||
|
// bucket bound again once and the act done once more on the new binding (H-A-251).
|
||||||
|
func inBucket[T any](c *Conn, ctx context.Context, s StateIssued, do func(jetstream.KeyValue) (T, error)) (T, error) {
|
||||||
|
var none T
|
||||||
|
kv, err := c.bucket(ctx, s)
|
||||||
|
if err != nil {
|
||||||
|
return none, err
|
||||||
|
}
|
||||||
|
got, err := do(kv)
|
||||||
|
if !bucketGone(err) {
|
||||||
|
return got, err
|
||||||
|
}
|
||||||
|
c.unbind(s.Bucket, kv)
|
||||||
|
if kv, err = c.bucket(ctx, s); err != nil {
|
||||||
|
return none, err
|
||||||
|
}
|
||||||
|
return do(kv)
|
||||||
}
|
}
|
||||||
|
|
||||||
func checkKey(key string) error {
|
func checkKey(key string) error {
|
||||||
@@ -130,11 +218,7 @@ func (c *Conn) StateGet(module, name, key string) (*StateEntry, error) {
|
|||||||
}
|
}
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
kv, err := c.bucket(ctx, s)
|
e, err := inBucket(c, ctx, s, func(kv jetstream.KeyValue) (jetstream.KeyValueEntry, error) { return kv.Get(ctx, key) })
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
e, err := kv.Get(ctx, key)
|
|
||||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
@@ -210,11 +294,10 @@ func (c *Conn) StatePage(module, name, after string, maxBytes int, budget time.D
|
|||||||
maps.Copy(current, got)
|
maps.Copy(current, got)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
kv, err := c.bucket(ctx, s)
|
var err error
|
||||||
if err != nil {
|
if current, err = inBucket(c, ctx, s, func(kv jetstream.KeyValue) (map[string]*StateEntry, error) {
|
||||||
return StatePage{}, err
|
return currentOf(ctx, kv)
|
||||||
}
|
}); err != nil {
|
||||||
if current, err = currentOf(ctx, kv); err != nil {
|
|
||||||
return StatePage{}, err
|
return StatePage{}, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -273,11 +356,9 @@ func (c *Conn) StateHistory(module, name, key string) ([]StateKept, error) {
|
|||||||
}
|
}
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
kv, err := c.bucket(ctx, s)
|
entries, err := inBucket(c, ctx, s, func(kv jetstream.KeyValue) ([]jetstream.KeyValueEntry, error) {
|
||||||
if err != nil {
|
return kv.History(ctx, key)
|
||||||
return nil, err
|
})
|
||||||
}
|
|
||||||
entries, err := kv.History(ctx, key)
|
|
||||||
kept := []StateKept{}
|
kept := []StateKept{}
|
||||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||||
return kept, nil
|
return kept, nil
|
||||||
@@ -326,11 +407,7 @@ func (c *Conn) StateInfo(module, name string) (*StateInfo, error) {
|
|||||||
}
|
}
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
kv, err := c.bucket(ctx, s)
|
status, err := inBucket(c, ctx, s, func(kv jetstream.KeyValue) (jetstream.KeyValueStatus, error) { return kv.Status(ctx) })
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
status, err := kv.Status(ctx)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -363,11 +440,7 @@ func (c *Conn) StatePut(module, name, key string, value json.RawMessage) (uint64
|
|||||||
}
|
}
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
kv, err := c.bucket(ctx, s)
|
return inBucket(c, ctx, s, func(kv jetstream.KeyValue) (uint64, error) { return kv.Put(ctx, key, value) })
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
return kv.Put(ctx, key, value)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// MinKeyTTL is the shortest time to live a key may be given: the bus counts a key's age in whole seconds (mesh-sdk
|
// MinKeyTTL is the shortest time to live a key may be given: the bus counts a key's age in whole seconds (mesh-sdk
|
||||||
@@ -437,11 +510,8 @@ func (c *Conn) StateDelete(module, name, key string) error {
|
|||||||
}
|
}
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
kv, err := c.bucket(ctx, s)
|
_, err = inBucket(c, ctx, s, func(kv jetstream.KeyValue) (struct{}, error) { return struct{}{}, kv.Delete(ctx, key) })
|
||||||
if err != nil {
|
return err
|
||||||
return err
|
|
||||||
}
|
|
||||||
return kv.Delete(ctx, key)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// StateKeys is every key with a current value, sorted.
|
// StateKeys is every key with a current value, sorted.
|
||||||
@@ -455,11 +525,7 @@ func (c *Conn) StateKeys(module, name string) ([]string, error) {
|
|||||||
}
|
}
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
kv, err := c.bucket(ctx, s)
|
return inBucket(c, ctx, s, func(kv jetstream.KeyValue) ([]string, error) { return keysOf(ctx, kv) })
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return keysOf(ctx, kv)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// keysOf is every key with a current value in one bucket, sorted.
|
// keysOf is every key with a current value in one bucket, sorted.
|
||||||
@@ -529,12 +595,16 @@ func (c *Conn) StateWatch(module, name, pattern string, deliver func(StateChange
|
|||||||
return c.perNodeWatch(module, s, pattern, deliver)
|
return c.perNodeWatch(module, s, pattern, deliver)
|
||||||
}
|
}
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||||
kv, err := c.bucket(ctx, s)
|
defer cancel()
|
||||||
cancel()
|
type watching struct{ stop func() }
|
||||||
|
w, err := inBucket(c, ctx, s, func(kv jetstream.KeyValue) (watching, error) {
|
||||||
|
stop, err := c.watchBucket(module, name, kv, filter, deliver)
|
||||||
|
return watching{stop}, err
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return c.watchBucket(module, name, kv, filter, deliver)
|
return w.stop, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// watchBucket is StateWatch over one bucket: the current values, then every change.
|
// watchBucket is StateWatch over one bucket: the current values, then every change.
|
||||||
|
|||||||
@@ -0,0 +1,81 @@
|
|||||||
|
//go:build integration
|
||||||
|
|
||||||
|
package bus_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **On the bus, a read of a state is one answer: its bucket is bound once, and bound again when it was made afresh or
|
||||||
|
// is gone** (design 47 H lead answers H-A-251). The runtime asked the bucket's stream info before every get, so each
|
||||||
|
// read was two answers; a third of a night's bus flood came this way.
|
||||||
|
func TestAStateReadOnTheBusBindsItsBucketOnce(t *testing.T) {
|
||||||
|
conn, kv := owedWatcher(t)
|
||||||
|
const bucket = "mesh-delivery-assigner_pins"
|
||||||
|
if _, err := kv.Put("a", []byte(`{"v":1}`)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
watcher, err := nats.Connect(mt.URL(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(watcher.Close)
|
||||||
|
var infos atomic.Int32
|
||||||
|
if _, err := watcher.Subscribe("$JS.API.STREAM.INFO.KV_"+bucket, func(*nats.Msg) { infos.Add(1) }); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := watcher.Flush(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
read := func() {
|
||||||
|
t.Helper()
|
||||||
|
e, err := conn.StateGet("gitea", "mesh-delivery-assigner.pins", "a")
|
||||||
|
if err != nil || e == nil {
|
||||||
|
t.Fatalf("read: %+v, %v", e, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
read()
|
||||||
|
read()
|
||||||
|
read()
|
||||||
|
if err := watcher.Flush(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := infos.Load(); got != 1 {
|
||||||
|
t.Fatalf("three reads asked the bucket's stream info %d times, want once", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The bucket deleted under the runtime: the read binds again, once, and says the bucket is not on the bus.
|
||||||
|
js, _ := watcher.JetStream()
|
||||||
|
if err := js.DeleteKeyValue(bucket); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_, err = conn.StateGet("gitea", "mesh-delivery-assigner.pins", "a")
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "not on the bus yet") {
|
||||||
|
t.Fatalf("a read of a deleted bucket answered %v, want that its bucket is not on the bus", err)
|
||||||
|
}
|
||||||
|
// And made afresh: read again from the new bucket.
|
||||||
|
again, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: bucket, History: 1})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := again.Put("a", []byte(`{"v":2}`)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
e, err := conn.StateGet("gitea", "mesh-delivery-assigner.pins", "a")
|
||||||
|
if err != nil || e == nil || string(e.Value) != `{"v":2}` {
|
||||||
|
t.Fatalf("a read of the bucket made afresh: %+v, %v", e, err)
|
||||||
|
}
|
||||||
|
read()
|
||||||
|
if err := watcher.Flush(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := infos.Load(); got != 3 {
|
||||||
|
t.Fatalf("the bucket's stream info was asked %d times in all, want 3: once, once more when it was gone, once when made afresh", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,141 @@
|
|||||||
|
package bus
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
)
|
||||||
|
|
||||||
|
// boundKV is a bucket's binding in a test: it answers every get with one value, or, once gone, as the bus answers a
|
||||||
|
// binding whose bucket was deleted.
|
||||||
|
type boundKV struct {
|
||||||
|
jetstream.KeyValue
|
||||||
|
gone atomic.Bool
|
||||||
|
puts atomic.Int32
|
||||||
|
}
|
||||||
|
|
||||||
|
type boundEntry struct {
|
||||||
|
jetstream.KeyValueEntry
|
||||||
|
key string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e boundEntry) Key() string { return e.key }
|
||||||
|
func (e boundEntry) Value() []byte { return []byte(`{"v":1}`) }
|
||||||
|
func (e boundEntry) Revision() uint64 { return 1 }
|
||||||
|
func (boundEntry) Created() time.Time { return time.Time{} }
|
||||||
|
func (*boundKV) Bucket() string { return "gitea_pins" }
|
||||||
|
func (kv *boundKV) Get(_ context.Context, key string) (jetstream.KeyValueEntry, error) {
|
||||||
|
if kv.gone.Load() {
|
||||||
|
return nil, jetstream.ErrStreamNotFound
|
||||||
|
}
|
||||||
|
return boundEntry{key: key}, nil
|
||||||
|
}
|
||||||
|
func (kv *boundKV) Put(context.Context, string, []byte) (uint64, error) {
|
||||||
|
if kv.gone.Load() {
|
||||||
|
return 0, jetstream.ErrNoStreamResponse
|
||||||
|
}
|
||||||
|
return uint64(kv.puts.Add(1)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A state is bound once for the process, and bound again only when the bus says its bucket is gone** (design 47 H
|
||||||
|
// lead answers H-A-251): binding is a stream-info request on the bus, and asked before every get it made each read two
|
||||||
|
// answers on the bus.
|
||||||
|
func TestAStateIsBoundOnceAndBoundAgainWhenItsBucketIsGone(t *testing.T) {
|
||||||
|
setup := func() (*Conn, *atomic.Int32, *[]*boundKV) {
|
||||||
|
var binds atomic.Int32
|
||||||
|
var mu sync.Mutex
|
||||||
|
made := []*boundKV{}
|
||||||
|
c := &Conn{node: "anchor", issued: map[string]*Membership{"gitea": {Module: "gitea",
|
||||||
|
State: []StateIssued{{Name: "pins", Bucket: "gitea_pins", Writes: true}}}}}
|
||||||
|
c.bind = func(_ context.Context, bucket string) (jetstream.KeyValue, error) {
|
||||||
|
binds.Add(1)
|
||||||
|
if bucket != "gitea_pins" {
|
||||||
|
t.Errorf("bound %s, want gitea_pins", bucket)
|
||||||
|
}
|
||||||
|
kv := &boundKV{}
|
||||||
|
mu.Lock()
|
||||||
|
made = append(made, kv)
|
||||||
|
mu.Unlock()
|
||||||
|
return kv, nil
|
||||||
|
}
|
||||||
|
return c, &binds, &made
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("two reads, one bind", func(t *testing.T) {
|
||||||
|
c, binds, _ := setup()
|
||||||
|
for range 2 {
|
||||||
|
if e, err := c.StateGet("gitea", "pins", "a"); err != nil || e == nil || e.Key != "a" {
|
||||||
|
t.Fatalf("read: %+v, %v", e, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := c.StatePut("gitea", "pins", "a", json.RawMessage(`{"v":2}`)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := binds.Load(); got != 1 {
|
||||||
|
t.Fatalf("two reads and a write bound the bucket %d times, want once", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("many readers at once, one bind", func(t *testing.T) {
|
||||||
|
c, binds, _ := setup()
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for range 16 {
|
||||||
|
wg.Go(func() {
|
||||||
|
if _, err := c.StateGet("gitea", "pins", "a"); err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
if got := binds.Load(); got != 1 {
|
||||||
|
t.Fatalf("16 readers bound the bucket %d times, want once", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("a bucket gone under the runner is bound again once", func(t *testing.T) {
|
||||||
|
c, binds, made := setup()
|
||||||
|
if _, err := c.StateGet("gitea", "pins", "a"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
(*made)[0].gone.Store(true)
|
||||||
|
if e, err := c.StateGet("gitea", "pins", "a"); err != nil || e == nil {
|
||||||
|
t.Fatalf("the read after the bucket was made again: %+v, %v", e, err)
|
||||||
|
}
|
||||||
|
if got := binds.Load(); got != 2 {
|
||||||
|
t.Fatalf("the bucket was bound %d times, want twice (once more after it was gone)", got)
|
||||||
|
}
|
||||||
|
if _, err := c.StateGet("gitea", "pins", "a"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
(*made)[1].gone.Store(true)
|
||||||
|
if _, err := c.StatePut("gitea", "pins", "a", json.RawMessage(`{"v":2}`)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got, puts := binds.Load(), (*made)[2].puts.Load(); got != 3 || puts != 1 {
|
||||||
|
t.Fatalf("a write to a gone bucket bound it %d times in all (want 3) and wrote %d times on the new binding (want 1)", got, puts)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("a bucket still gone after the bind again is said", func(t *testing.T) {
|
||||||
|
c, binds, made := setup()
|
||||||
|
if _, err := c.StateGet("gitea", "pins", "a"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
(*made)[0].gone.Store(true)
|
||||||
|
c.bind = func(context.Context, string) (jetstream.KeyValue, error) {
|
||||||
|
binds.Add(1)
|
||||||
|
return nil, jetstream.ErrBucketNotFound
|
||||||
|
}
|
||||||
|
if _, err := c.StateGet("gitea", "pins", "a"); err == nil {
|
||||||
|
t.Fatal("a read of a bucket no longer on the bus answered no error")
|
||||||
|
}
|
||||||
|
if got := binds.Load(); got != 2 {
|
||||||
|
t.Fatalf("the bucket was bound %d times, want twice: the bind again is once", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -10,6 +10,8 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/nats-io/nats.go/jetstream"
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/process"
|
||||||
)
|
)
|
||||||
|
|
||||||
// A module's own stream, as the runtime publishes to it and hears it (novox/hq ADR 0363 §8, issue 725).
|
// A module's own stream, as the runtime publishes to it and hears it (novox/hq ADR 0363 §8, issue 725).
|
||||||
@@ -129,7 +131,10 @@ func (h *hearing) bind(stream, durable string, onErr func(error)) (jetstream.Con
|
|||||||
if onErr != nil {
|
if onErr != nil {
|
||||||
opts = append(opts, jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { onErr(err) }))
|
opts = append(opts, jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { onErr(err) }))
|
||||||
}
|
}
|
||||||
reading, err := consumer.Consume(h.handle, opts...)
|
// Claimed through the SDK's one guard as it is pulled (design 47 H, H-A-196, H-A-197): a second pull loop on a durable
|
||||||
|
// this process pulls already would take every other event from the first, so it is refused here, loud. The claim is
|
||||||
|
// released as the loop is stopped, or once the bus ended it, before its Closed is closed.
|
||||||
|
reading, err := process.ConsumeOnce(consumer, "the tool runner's hearing of "+h.module, h.handle, opts...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, time.Time{}, "reading", err
|
return nil, time.Time{}, "reading", err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
)
|
)
|
||||||
|
|
||||||
// novox/hq ADR 0363 §2, §3: an event goes to the stream whose pattern carries it, as the bus matches subjects.
|
// novox/hq ADR 0363 §2, §3: an event goes to the stream whose pattern carries it, as the bus matches subjects.
|
||||||
@@ -69,3 +70,19 @@ func TestAnEventNoStreamCarriesIsPublishedOnEvents(t *testing.T) {
|
|||||||
t.Errorf("a stream's event reaches its consumer as %q, not as on EVENTS", got)
|
t.Errorf("a stream's event reaches its consumer as %q, not as on EVENTS", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Design 47 H lead answers H-A-227: an object store's fill is the figures the bus keeps for its stream, its live
|
||||||
|
// subjects as the count, never a count of its descriptions: a store holding twenty thousand messages, deleted chunks'
|
||||||
|
// leftovers among them, answers its bytes, its cap and its four live subjects.
|
||||||
|
func TestAnObjectStoresFillIsItsStreamsOwnFigures(t *testing.T) {
|
||||||
|
info := &jetstream.StreamInfo{
|
||||||
|
Config: jetstream.StreamConfig{MaxBytes: 8 << 30},
|
||||||
|
State: jetstream.StreamState{Msgs: 20653, Bytes: 1 << 30, NumSubjects: 4, FirstSeq: 9, LastSeq: 20661},
|
||||||
|
}
|
||||||
|
if got, want := objectsFillOf(info), (buscalls.Fill{Count: 4, Bytes: 1 << 30, MaxBytes: 8 << 30}); got != want {
|
||||||
|
t.Fatalf("fill %+v, want %+v", got, want)
|
||||||
|
}
|
||||||
|
if got := objectsFillOf(&jetstream.StreamInfo{State: jetstream.StreamState{Bytes: 7}}); got != (buscalls.Fill{Bytes: 7}) {
|
||||||
|
t.Fatalf("a store with no cap and no live subject: fill %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -258,6 +258,34 @@ func TestAConsumerNamedBeforeItExistsIsBoundOnceItDoes(t *testing.T) {
|
|||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// A second pull loop on the durable the runner pulls is refused, naming it and both binders (design 47 H, H-A-196,
|
||||||
|
// H-A-197); a re-serve that stops its loop first binds again without a refusal, its module stream's durable too.
|
||||||
|
if _, err := conn.ConsumeAs("reader", func(bus.Envelope) error { return nil }); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "pulled in this process already") || !strings.Contains(err.Error(), "anchor_reader") ||
|
||||||
|
strings.Count(err.Error(), "the tool runner's hearing of reader") != 2 {
|
||||||
|
t.Fatalf("a second pull loop on the durable the runner pulls was not refused naming it and both binders: %v", err)
|
||||||
|
}
|
||||||
|
stop()
|
||||||
|
again, err := conn.ConsumeAs("reader", func(env bus.Envelope) error {
|
||||||
|
mu.Lock()
|
||||||
|
heard[env.Key]++
|
||||||
|
mu.Unlock()
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("the re-serve was refused after its old loop stopped: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(again)
|
||||||
|
mesh.Emit(t, "mesh.stream.mail.event.mail.home.message.added", map[string]any{})
|
||||||
|
mt.Until(t, func() error {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
if heard["mail.mail.home.message.added"] != 2 {
|
||||||
|
return fmt.Errorf("after the re-serve heard %v", heard)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// novox/hq ADR 0363 §6 (issue 725): a consumer the controller deletes and makes again under the same name, its
|
// novox/hq ADR 0363 §6 (issue 725): a consumer the controller deletes and makes again under the same name, its
|
||||||
|
|||||||
Reference in New Issue
Block a user