Author SHA1 Message Date
mesh-tools d064bb75cd Merge pull request 'Design 47 H1, proof 2: one comment in the tool runner, delivered with the SDK in one merge call' (#83) from proof/47-H1-tools into main 2026-11-21 11:27:08 +00:00
mesh-tools ee143c9f91 Merge pull request 'Design 47 H1: the tool runner binds a state once per process (H-A-251)' (#85) from fix/47-H1-tools-bind-once into main 2026-11-21 11:01:54 +00:00
jschoubben 05d8f6b762 H-A-251: bind a state's bucket once for the process, not before every read
The runtime asked the bucket's stream info before every get, put, delete and
listing, so each state read was two answers on the bus (a third of the night's
flood came this way, ten thousand reads per sweep). A Conn is one account's
connection; it now keeps one binding per bucket, shared by concurrent askers,
and binds again once when the bus says the bucket is gone.

Builds: design 47 H H-A-251
2026-11-21 11:55:35 +01:00
mesh-tools fe707779e3 Merge pull request 'Design 47 H1: an object store's fill is the stream's own figures, never a listing (H-A-227)' (#84) from fix/47-H1-tools-fill-figures into main 2026-11-21 01:12:17 +00:00
jschoubben 3eef0254df H-A-227: answer an object store's fill from its stream's own figures
The fill was a listing of every description in the store, deleted chunks'
leftovers included; a receiver reading it per chunk made one connection read
about 14 MB/s. The fill is now one stream info request: its bytes, its cap,
and its live subjects as the count.

Builds: design 47 H H-A-227
2026-11-21 01:59:38 +01:00
jschoubben 39c2604926 Design 47 H1, proof 2: mark the importer side of the two-module change
The bus package's doc names the proof, so node-tools rebuilds against the SDK version the same merge call publishes; go.mod keeps mesh-sdk at the zero version.

Builds: design 47 H §5 step 7, G §370
2026-11-21 01:46:34 +01:00
mesh-tools 07f5aa11c3 Merge pull request 'Design 47 H1: the tool runner claims each durable it pulls through the SDK's guard (H-A-197)' (#82) from fix/47-H1-runner-pull-once into main 2026-11-20 23:53:13 +00:00
jschoubben 876adfbf05 H-A-197: the runner pulls through the SDK's ConsumeOnce, keeping no copy
Review round 1 (P7): the runner's bind copied the SDK's claim wrapper and its
caller lookup. It now calls process.ConsumeOnce, which claims, pulls and
releases (at a stop, or once a drain or the bus ended the loop, before its
Closed is closed), naming the claim for the module whose hearing it is.

Builds: design 47 H H-A-197, H-A-196
2026-11-20 22:20:07 +01:00
jschoubben c952386737 H-A-197: the tool runner claims each durable it pulls through the SDK's guard
A tool runner built before H-A-132/133 ran two pull loops on the indexer
bundle's one durable, each event reached one loop only, and nothing said so;
the SDK's guard (H-A-196) could not see it, since the runner bound through the
bus client directly. hearing.bind now claims each durable with
process.PullOnce, naming the runner's hearing and the part that asked it, and
releases the claim as the loop is stopped or drained, or as the bus ends it,
so a re-serve (which stops its old loop first) binds again without a refusal.
The heartbeat test's second consume pulled the same durable: it gets its own.

Builds: design 47 H H-A-197, H-A-196
2026-11-20 22:12:45 +01:00
12 changed files with 440 additions and 68 deletions
-1
View File
@@ -1,6 +1,5 @@
// node-tools — the node's tool runtime, in Go (novox/hq ADR 0175, ADR 0193; to-be 38 WP4d). // node-tools — the node's tool runtime, in Go (novox/hq ADR 0175, ADR 0193; to-be 38 WP4d).
// (Its pipeline is asked again by this line, 2026-11-16 23:45: main.981 passed all but composition, whose revision 433 was removed; the assigner now waits 3 minutes for composing nodes.) // (Its pipeline is asked again by this line, 2026-11-16 23:45: main.981 passed all but composition, whose revision 433 was removed; the assigner now waits 3 minutes for composing nodes.)
// Design 47 H1, proof 2 once more: the importer half of the edge mesh-delivery-sdk to node-tools.
// //
// One process per machine: it connects to the bus on the node's credential, launches every assigned // One process per machine: it connects to the bus on the node's credential, launches every assigned
// module's tools bundle and serves its tools and its seats' verbs; as the node-tools module — or // module's tools bundle and serves its tools and its seats' verbs; as the node-tools module — or
+33 -14
View File
@@ -196,19 +196,28 @@ func (c *Conn) ObjectsList(module string, r buscalls.ObjectsRequest) (buscalls.O
return out, nil return out, nil
} }
// ObjectsInfo is how full the module's object store is: its objects, its bytes and its cap. // ObjectsInfo is how full the module's object store is, from the figures the bus keeps for its stream: its bytes, its
// messages' live subjects as the count, and its cap. It never lists the store: a store's descriptions, the leftovers
// of deleted objects among them, may be many thousands, and a fill read per chunk of a transfer read them all each
// time (design 47 H lead answers H-A-227).
func (c *Conn) ObjectsInfo(module string, r buscalls.ObjectsRequest) (buscalls.Fill, error) { func (c *Conn) ObjectsInfo(module string, r buscalls.ObjectsRequest) (buscalls.Fill, error) {
listed, err := c.ObjectsList(module, r) b, err := c.objectBucketOf(module, r.Objects)
if err != nil { if err != nil {
return buscalls.Fill{}, err return buscalls.Fill{}, err
} }
b, _ := c.objectBucketOf(module, r.Objects) info, err := c.streamInfo("OBJ_" + b.ObjectStore)
fill, err := c.streamFill("OBJ_" + b.ObjectStore)
if err != nil { if err != nil {
return buscalls.Fill{}, err return buscalls.Fill{}, err
} }
fill.Count = uint64(len(listed.Objects)) return objectsFillOf(info), nil
return fill.Fill, nil }
// objectsFillOf is an object store's fill from its stream's info alone: the bytes the stream keeps, its live subjects
// as the count (H-A-227: the stream's live subjects, or none) and its cap.
func objectsFillOf(info *jetstream.StreamInfo) buscalls.Fill {
fill := streamFillOf(info).Fill
fill.Count = uint64(info.State.NumSubjects)
return fill
} }
// ObjectsWatch hands deliver every object of the module's object store as it is, then each stored or deleted. // ObjectsWatch hands deliver every object of the module's object store as it is, then each stored or deleted.
@@ -261,29 +270,39 @@ func (c *Conn) ObjectsWatch(module string, r buscalls.ObjectsWatchRequest, deliv
// streamFill is a stream's fill and its first and last sequence, by its name on the bus. // streamFill is a stream's fill and its first and last sequence, by its name on the bus.
func (c *Conn) streamFill(name string) (buscalls.StreamFilled, error) { func (c *Conn) streamFill(name string) (buscalls.StreamFilled, error) {
info, err := c.streamInfo(name)
if err != nil {
return buscalls.StreamFilled{}, err
}
return streamFillOf(info), nil
}
// streamInfo is a stream's info as the bus keeps it, by its name on the bus: one request, whatever the stream holds.
func (c *Conn) streamInfo(name string) (*jetstream.StreamInfo, error) {
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
defer cancel() defer cancel()
js, err := c.JetStream() js, err := c.JetStream()
if err != nil { if err != nil {
return buscalls.StreamFilled{}, err return nil, err
} }
s, err := js.Stream(ctx, name) s, err := js.Stream(ctx, name)
if errors.Is(err, jetstream.ErrStreamNotFound) { if errors.Is(err, jetstream.ErrStreamNotFound) {
return buscalls.StreamFilled{}, fmt.Errorf("the stream %s is bound and not on the bus", name) return nil, fmt.Errorf("the stream %s is bound and not on the bus", name)
} }
if err != nil { if err != nil {
return buscalls.StreamFilled{}, err return nil, err
}
info, err := s.Info(ctx)
if err != nil {
return buscalls.StreamFilled{}, err
} }
return s.Info(ctx)
}
// streamFillOf is a stream's fill and its first and last sequence from its info.
func streamFillOf(info *jetstream.StreamInfo) buscalls.StreamFilled {
max := uint64(0) max := uint64(0)
if info.Config.MaxBytes > 0 { if info.Config.MaxBytes > 0 {
max = uint64(info.Config.MaxBytes) max = uint64(info.Config.MaxBytes)
} }
return buscalls.StreamFilled{Fill: buscalls.Fill{Count: info.State.Msgs, Bytes: info.State.Bytes, MaxBytes: max}, return buscalls.StreamFilled{Fill: buscalls.Fill{Count: info.State.Msgs, Bytes: info.State.Bytes, MaxBytes: max},
First: info.State.FirstSeq, Last: info.State.LastSeq}, nil First: info.State.FirstSeq, Last: info.State.LastSeq}
} }
// streamBinding is the module's binding of a stream by its local name, a work queue among them. // streamBinding is the module's binding of a stream by its local name, a work queue among them.
+5
View File
@@ -95,6 +95,11 @@ func TestABundleReachesItsOwnBoundObjectsAndStreamsOnly(t *testing.T) {
if listed, err := conn.ObjectsList("m", buscalls.ObjectsRequest{Objects: "files"}); err != nil || len(listed.Objects) != 1 { if listed, err := conn.ObjectsList("m", buscalls.ObjectsRequest{Objects: "files"}); err != nil || len(listed.Objects) != 1 {
t.Fatalf("listed %+v, %v", listed, err) t.Fatalf("listed %+v, %v", listed, err)
} }
// H-A-227: the fill is the stream's own figures, its live subjects as the count (a's chunk and description, big's
// description left by its delete), never the count of its listed objects.
if fill, err := conn.ObjectsInfo("m", buscalls.ObjectsRequest{Objects: "files"}); err != nil || fill.Count != 3 || fill.Bytes == 0 {
t.Fatalf("objects fill %+v, %v", fill, err)
}
// A-380: an old `objects` entry is reached by the store grants creates for it, `<module>_<name>`, with no binding of that name. // A-380: an old `objects` entry is reached by the store grants creates for it, `<module>_<name>`, with no binding of that name.
if _, err := conn.ObjectsPut("m", buscalls.ObjectsPutRequest{Objects: "old", Name: "r", Data: []byte("old section")}); err != nil { if _, err := conn.ObjectsPut("m", buscalls.ObjectsPutRequest{Objects: "old", Name: "r", Data: []byte("old section")}); err != nil {
t.Fatalf("put into the old objects section: %v", err) t.Fatalf("put into the old objects section: %v", err)
+9
View File
@@ -7,6 +7,8 @@
// mesh.seat.<seat>.tool.<verb> a role's verb, answered by whoever holds the seat // mesh.seat.<seat>.tool.<verb> a role's verb, answered by whoever holds the seat
// mesh.mod.<module>.call.<tool>.<caller>, mesh.seat.<seat>.call.<verb>.<caller> // mesh.mod.<module>.call.<tool>.<caller>, mesh.seat.<seat>.call.<verb>.<caller>
// the same, naming the bus user that called (caller.go, hq issue 365) // the same, naming the bus user that called (caller.go, hq issue 365)
//
// Design 47 H1, proof 2: the SDK and its importer delivered in one merge call, 2026-11-21.
package bus package bus
import ( import (
@@ -203,6 +205,13 @@ type Conn struct {
waits map[string]map[chan string]bool waits map[string]map[chan string]bool
// inboxSaid is when a refused answer publish was last said, by subject (answerRefused). // inboxSaid is when a refused answer publish was last said, by subject (answerRefused).
inboxSaid map[string]time.Time inboxSaid map[string]time.Time
// bound is each state's binding, kept for the process and bound again only when the bus says its bucket is gone
// (state.go, design 47 H lead answers H-A-251): one Conn is one account's connection, so a binding is per account
// and bucket.
bindMu sync.Mutex
bound map[string]*binding
// bind binds one bucket: jetstream's KeyValue (a stream-info request on the bus) unless a test replaces it.
bind func(ctx context.Context, bucket string) (jetstream.KeyValue, error)
} }
// answered is one subject the runtime answers on: what to subscribe again with, and how often it was // answered is one subject the runtime answers on: what to subscribe again with, and how often it was
+3 -1
View File
@@ -77,6 +77,8 @@ func TestAConsumerSaysItIsBoundWhileIdleAndOnceWhenTheBusEndedIt(t *testing.T) {
} }
})) }))
mesh.Consumer(t, "anchor", "quiet", []string{"mesh.mod.alpha.event.>"}, time.Second) mesh.Consumer(t, "anchor", "quiet", []string{"mesh.mod.alpha.event.>"}, time.Second)
// The unasked consume below pulls a durable of its own: a second loop on quiet's is refused (H-A-196, H-A-197).
mesh.Consumer(t, "anchor", "unasked", []string{"mesh.mod.alpha.event.>"}, time.Second)
c, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: "node-tools", Node: "anchor"}) c, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: "node-tools", Node: "anchor"})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@@ -97,7 +99,7 @@ func TestAConsumerSaysItIsBoundWhileIdleAndOnceWhenTheBusEndedIt(t *testing.T) {
t.Cleanup(stop) t.Cleanup(stop)
// A consume nobody asked a heartbeat of says nothing and asks the bus nothing, however many intervals pass. // A consume nobody asked a heartbeat of says nothing and asks the bus nothing, however many intervals pass.
var unaskedBeats atomic.Int32 var unaskedBeats atomic.Int32
unasked, err := c.ConsumeAsBeatingWhen("quiet", func(bus.Envelope) error { return nil }, unasked, err := c.ConsumeAsBeatingWhen("unasked", func(bus.Envelope) error { return nil },
func(bus.Beat) { unaskedBeats.Add(1) }, func() bool { return false }) func(bus.Beat) { unaskedBeats.Add(1) }, func() bool { return false })
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
+1 -5
View File
@@ -427,11 +427,7 @@ func (c *Conn) stateWrite(module, name, key string, value json.RawMessage,
} }
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
defer cancel() defer cancel()
kv, err := c.bucket(ctx, s) rev, err := inBucket(c, ctx, s, func(kv jetstream.KeyValue) (uint64, error) { return write(ctx, kv) })
if err != nil {
return 0, err
}
rev, err := write(ctx, kv)
if errors.Is(err, jetstream.ErrKeyExists) || isWrongSequence(err) { if errors.Is(err, jetstream.ErrKeyExists) || isWrongSequence(err) {
return 0, changed{name + "." + key} return 0, changed{name + "." + key}
} }
+113 -43
View File
@@ -15,6 +15,7 @@ import (
"git.novox.be/novox/mesh-sdk/go/membership" "git.novox.be/novox/mesh-sdk/go/membership"
"git.novox.be/novox/mesh-sdk/go/pernode" "git.novox.be/novox/mesh-sdk/go/pernode"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream" "github.com/nats-io/nats.go/jetstream"
) )
@@ -92,20 +93,107 @@ func (c *Conn) issuedState(module, name string) (StateIssued, error) {
"issued: %s (novox/hq ADR 0201, ADR 0595 §1)", module, name, issued) "issued: %s (novox/hq ADR 0201, ADR 0595 §1)", module, name, issued)
} }
// binding is one bucket's binding, or the bind in flight for it: the first asker binds, an asker meanwhile waits for
// that bind (done) instead of asking the bus again.
type binding struct {
done chan struct{}
kv jetstream.KeyValue
err error
}
// bucket is the state's binding, bound once for the process (design 47 H lead answers H-A-251): binding a bucket is a
// stream-info request on the bus, and asked before every get it made each read two answers. A bind that fails is not
// kept, so the next ask binds again.
func (c *Conn) bucket(ctx context.Context, s StateIssued) (jetstream.KeyValue, error) { func (c *Conn) bucket(ctx context.Context, s StateIssued) (jetstream.KeyValue, error) {
c.bindMu.Lock()
b := c.bound[s.Bucket]
if b == nil {
if c.bound == nil {
c.bound = map[string]*binding{}
}
b = &binding{done: make(chan struct{})}
c.bound[s.Bucket] = b
c.bindMu.Unlock()
b.kv, b.err = c.bindBucket(ctx, s.Bucket)
if b.err != nil {
c.bindMu.Lock()
if c.bound[s.Bucket] == b {
delete(c.bound, s.Bucket)
}
c.bindMu.Unlock()
}
close(b.done)
} else {
c.bindMu.Unlock()
select {
case <-b.done:
case <-ctx.Done():
return nil, ctx.Err()
}
}
if b.err != nil {
if errors.Is(b.err, jetstream.ErrBucketNotFound) {
return nil, fmt.Errorf("the state %q is issued and its bucket is not on the bus yet: the mesh "+
"creates it from the catalogue on its next raise", s.Name)
}
return nil, b.err
}
return b.kv, nil
}
func (c *Conn) bindBucket(ctx context.Context, bucket string) (jetstream.KeyValue, error) {
if c.bind != nil {
return c.bind(ctx, bucket)
}
js, err := jetstream.New(c.nc) js, err := jetstream.New(c.nc)
if err != nil { if err != nil {
return nil, err return nil, err
} }
kv, err := js.KeyValue(ctx, s.Bucket) return js.KeyValue(ctx, bucket)
}
// unbind drops a bucket's binding the bus said is gone, when it is still the one kept, so the next ask binds again.
func (c *Conn) unbind(bucket string, kv jetstream.KeyValue) {
c.bindMu.Lock()
defer c.bindMu.Unlock()
b := c.bound[bucket]
if b == nil {
return
}
select {
case <-b.done: // bound: its kv is written
default:
return // bound again already, in flight
}
if b.kv == kv {
delete(c.bound, bucket)
}
}
// bucketGone is the bus saying a kept binding's bucket is not there (deleted, or deleted and created again): no stream of
// the name, or nobody answers on its subjects.
func bucketGone(err error) bool {
return errors.Is(err, jetstream.ErrStreamNotFound) || errors.Is(err, jetstream.ErrBucketNotFound) ||
errors.Is(err, jetstream.ErrNoStreamResponse) || errors.Is(err, nats.ErrNoResponders)
}
// inBucket does one act on the state's kept binding; when the bus says its bucket is gone, the binding is dropped, the
// bucket bound again once and the act done once more on the new binding (H-A-251).
func inBucket[T any](c *Conn, ctx context.Context, s StateIssued, do func(jetstream.KeyValue) (T, error)) (T, error) {
var none T
kv, err := c.bucket(ctx, s)
if err != nil { if err != nil {
if errors.Is(err, jetstream.ErrBucketNotFound) { return none, err
return nil, fmt.Errorf("the state %q is issued and its bucket is not on the bus yet: the mesh "+
"creates it from the catalogue on its next raise", s.Name)
} }
return nil, err got, err := do(kv)
if !bucketGone(err) {
return got, err
} }
return kv, nil c.unbind(s.Bucket, kv)
if kv, err = c.bucket(ctx, s); err != nil {
return none, err
}
return do(kv)
} }
func checkKey(key string) error { func checkKey(key string) error {
@@ -130,11 +218,7 @@ func (c *Conn) StateGet(module, name, key string) (*StateEntry, error) {
} }
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
defer cancel() defer cancel()
kv, err := c.bucket(ctx, s) e, err := inBucket(c, ctx, s, func(kv jetstream.KeyValue) (jetstream.KeyValueEntry, error) { return kv.Get(ctx, key) })
if err != nil {
return nil, err
}
e, err := kv.Get(ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) { if errors.Is(err, jetstream.ErrKeyNotFound) {
return nil, nil return nil, nil
} }
@@ -210,11 +294,10 @@ func (c *Conn) StatePage(module, name, after string, maxBytes int, budget time.D
maps.Copy(current, got) maps.Copy(current, got)
} }
} else { } else {
kv, err := c.bucket(ctx, s) var err error
if err != nil { if current, err = inBucket(c, ctx, s, func(kv jetstream.KeyValue) (map[string]*StateEntry, error) {
return StatePage{}, err return currentOf(ctx, kv)
} }); err != nil {
if current, err = currentOf(ctx, kv); err != nil {
return StatePage{}, err return StatePage{}, err
} }
} }
@@ -273,11 +356,9 @@ func (c *Conn) StateHistory(module, name, key string) ([]StateKept, error) {
} }
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
defer cancel() defer cancel()
kv, err := c.bucket(ctx, s) entries, err := inBucket(c, ctx, s, func(kv jetstream.KeyValue) ([]jetstream.KeyValueEntry, error) {
if err != nil { return kv.History(ctx, key)
return nil, err })
}
entries, err := kv.History(ctx, key)
kept := []StateKept{} kept := []StateKept{}
if errors.Is(err, jetstream.ErrKeyNotFound) { if errors.Is(err, jetstream.ErrKeyNotFound) {
return kept, nil return kept, nil
@@ -326,11 +407,7 @@ func (c *Conn) StateInfo(module, name string) (*StateInfo, error) {
} }
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
defer cancel() defer cancel()
kv, err := c.bucket(ctx, s) status, err := inBucket(c, ctx, s, func(kv jetstream.KeyValue) (jetstream.KeyValueStatus, error) { return kv.Status(ctx) })
if err != nil {
return nil, err
}
status, err := kv.Status(ctx)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -363,11 +440,7 @@ func (c *Conn) StatePut(module, name, key string, value json.RawMessage) (uint64
} }
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
defer cancel() defer cancel()
kv, err := c.bucket(ctx, s) return inBucket(c, ctx, s, func(kv jetstream.KeyValue) (uint64, error) { return kv.Put(ctx, key, value) })
if err != nil {
return 0, err
}
return kv.Put(ctx, key, value)
} }
// MinKeyTTL is the shortest time to live a key may be given: the bus counts a key's age in whole seconds (mesh-sdk // MinKeyTTL is the shortest time to live a key may be given: the bus counts a key's age in whole seconds (mesh-sdk
@@ -437,11 +510,8 @@ func (c *Conn) StateDelete(module, name, key string) error {
} }
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
defer cancel() defer cancel()
kv, err := c.bucket(ctx, s) _, err = inBucket(c, ctx, s, func(kv jetstream.KeyValue) (struct{}, error) { return struct{}{}, kv.Delete(ctx, key) })
if err != nil {
return err return err
}
return kv.Delete(ctx, key)
} }
// StateKeys is every key with a current value, sorted. // StateKeys is every key with a current value, sorted.
@@ -455,11 +525,7 @@ func (c *Conn) StateKeys(module, name string) ([]string, error) {
} }
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
defer cancel() defer cancel()
kv, err := c.bucket(ctx, s) return inBucket(c, ctx, s, func(kv jetstream.KeyValue) ([]string, error) { return keysOf(ctx, kv) })
if err != nil {
return nil, err
}
return keysOf(ctx, kv)
} }
// keysOf is every key with a current value in one bucket, sorted. // keysOf is every key with a current value in one bucket, sorted.
@@ -529,12 +595,16 @@ func (c *Conn) StateWatch(module, name, pattern string, deliver func(StateChange
return c.perNodeWatch(module, s, pattern, deliver) return c.perNodeWatch(module, s, pattern, deliver)
} }
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout) ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
kv, err := c.bucket(ctx, s) defer cancel()
cancel() type watching struct{ stop func() }
w, err := inBucket(c, ctx, s, func(kv jetstream.KeyValue) (watching, error) {
stop, err := c.watchBucket(module, name, kv, filter, deliver)
return watching{stop}, err
})
if err != nil { if err != nil {
return nil, err return nil, err
} }
return c.watchBucket(module, name, kv, filter, deliver) return w.stop, nil
} }
// watchBucket is StateWatch over one bucket: the current values, then every change. // watchBucket is StateWatch over one bucket: the current values, then every change.
@@ -0,0 +1,81 @@
//go:build integration
package bus_test
import (
"strings"
"sync/atomic"
"testing"
"github.com/nats-io/nats.go"
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
)
// **On the bus, a read of a state is one answer: its bucket is bound once, and bound again when it was made afresh or
// is gone** (design 47 H lead answers H-A-251). The runtime asked the bucket's stream info before every get, so each
// read was two answers; a third of a night's bus flood came this way.
func TestAStateReadOnTheBusBindsItsBucketOnce(t *testing.T) {
conn, kv := owedWatcher(t)
const bucket = "mesh-delivery-assigner_pins"
if _, err := kv.Put("a", []byte(`{"v":1}`)); err != nil {
t.Fatal(err)
}
watcher, err := nats.Connect(mt.URL(t))
if err != nil {
t.Fatal(err)
}
t.Cleanup(watcher.Close)
var infos atomic.Int32
if _, err := watcher.Subscribe("$JS.API.STREAM.INFO.KV_"+bucket, func(*nats.Msg) { infos.Add(1) }); err != nil {
t.Fatal(err)
}
if err := watcher.Flush(); err != nil {
t.Fatal(err)
}
read := func() {
t.Helper()
e, err := conn.StateGet("gitea", "mesh-delivery-assigner.pins", "a")
if err != nil || e == nil {
t.Fatalf("read: %+v, %v", e, err)
}
}
read()
read()
read()
if err := watcher.Flush(); err != nil {
t.Fatal(err)
}
if got := infos.Load(); got != 1 {
t.Fatalf("three reads asked the bucket's stream info %d times, want once", got)
}
// The bucket deleted under the runtime: the read binds again, once, and says the bucket is not on the bus.
js, _ := watcher.JetStream()
if err := js.DeleteKeyValue(bucket); err != nil {
t.Fatal(err)
}
_, err = conn.StateGet("gitea", "mesh-delivery-assigner.pins", "a")
if err == nil || !strings.Contains(err.Error(), "not on the bus yet") {
t.Fatalf("a read of a deleted bucket answered %v, want that its bucket is not on the bus", err)
}
// And made afresh: read again from the new bucket.
again, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: bucket, History: 1})
if err != nil {
t.Fatal(err)
}
if _, err := again.Put("a", []byte(`{"v":2}`)); err != nil {
t.Fatal(err)
}
e, err := conn.StateGet("gitea", "mesh-delivery-assigner.pins", "a")
if err != nil || e == nil || string(e.Value) != `{"v":2}` {
t.Fatalf("a read of the bucket made afresh: %+v, %v", e, err)
}
read()
if err := watcher.Flush(); err != nil {
t.Fatal(err)
}
if got := infos.Load(); got != 3 {
t.Fatalf("the bucket's stream info was asked %d times in all, want 3: once, once more when it was gone, once when made afresh", got)
}
}
+141
View File
@@ -0,0 +1,141 @@
package bus
import (
"context"
"encoding/json"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/nats-io/nats.go/jetstream"
)
// boundKV is a bucket's binding in a test: it answers every get with one value, or, once gone, as the bus answers a
// binding whose bucket was deleted.
type boundKV struct {
jetstream.KeyValue
gone atomic.Bool
puts atomic.Int32
}
type boundEntry struct {
jetstream.KeyValueEntry
key string
}
func (e boundEntry) Key() string { return e.key }
func (e boundEntry) Value() []byte { return []byte(`{"v":1}`) }
func (e boundEntry) Revision() uint64 { return 1 }
func (boundEntry) Created() time.Time { return time.Time{} }
func (*boundKV) Bucket() string { return "gitea_pins" }
func (kv *boundKV) Get(_ context.Context, key string) (jetstream.KeyValueEntry, error) {
if kv.gone.Load() {
return nil, jetstream.ErrStreamNotFound
}
return boundEntry{key: key}, nil
}
func (kv *boundKV) Put(context.Context, string, []byte) (uint64, error) {
if kv.gone.Load() {
return 0, jetstream.ErrNoStreamResponse
}
return uint64(kv.puts.Add(1)), nil
}
// **A state is bound once for the process, and bound again only when the bus says its bucket is gone** (design 47 H
// lead answers H-A-251): binding is a stream-info request on the bus, and asked before every get it made each read two
// answers on the bus.
func TestAStateIsBoundOnceAndBoundAgainWhenItsBucketIsGone(t *testing.T) {
setup := func() (*Conn, *atomic.Int32, *[]*boundKV) {
var binds atomic.Int32
var mu sync.Mutex
made := []*boundKV{}
c := &Conn{node: "anchor", issued: map[string]*Membership{"gitea": {Module: "gitea",
State: []StateIssued{{Name: "pins", Bucket: "gitea_pins", Writes: true}}}}}
c.bind = func(_ context.Context, bucket string) (jetstream.KeyValue, error) {
binds.Add(1)
if bucket != "gitea_pins" {
t.Errorf("bound %s, want gitea_pins", bucket)
}
kv := &boundKV{}
mu.Lock()
made = append(made, kv)
mu.Unlock()
return kv, nil
}
return c, &binds, &made
}
t.Run("two reads, one bind", func(t *testing.T) {
c, binds, _ := setup()
for range 2 {
if e, err := c.StateGet("gitea", "pins", "a"); err != nil || e == nil || e.Key != "a" {
t.Fatalf("read: %+v, %v", e, err)
}
}
if _, err := c.StatePut("gitea", "pins", "a", json.RawMessage(`{"v":2}`)); err != nil {
t.Fatal(err)
}
if got := binds.Load(); got != 1 {
t.Fatalf("two reads and a write bound the bucket %d times, want once", got)
}
})
t.Run("many readers at once, one bind", func(t *testing.T) {
c, binds, _ := setup()
var wg sync.WaitGroup
for range 16 {
wg.Go(func() {
if _, err := c.StateGet("gitea", "pins", "a"); err != nil {
t.Error(err)
}
})
}
wg.Wait()
if got := binds.Load(); got != 1 {
t.Fatalf("16 readers bound the bucket %d times, want once", got)
}
})
t.Run("a bucket gone under the runner is bound again once", func(t *testing.T) {
c, binds, made := setup()
if _, err := c.StateGet("gitea", "pins", "a"); err != nil {
t.Fatal(err)
}
(*made)[0].gone.Store(true)
if e, err := c.StateGet("gitea", "pins", "a"); err != nil || e == nil {
t.Fatalf("the read after the bucket was made again: %+v, %v", e, err)
}
if got := binds.Load(); got != 2 {
t.Fatalf("the bucket was bound %d times, want twice (once more after it was gone)", got)
}
if _, err := c.StateGet("gitea", "pins", "a"); err != nil {
t.Fatal(err)
}
(*made)[1].gone.Store(true)
if _, err := c.StatePut("gitea", "pins", "a", json.RawMessage(`{"v":2}`)); err != nil {
t.Fatal(err)
}
if got, puts := binds.Load(), (*made)[2].puts.Load(); got != 3 || puts != 1 {
t.Fatalf("a write to a gone bucket bound it %d times in all (want 3) and wrote %d times on the new binding (want 1)", got, puts)
}
})
t.Run("a bucket still gone after the bind again is said", func(t *testing.T) {
c, binds, made := setup()
if _, err := c.StateGet("gitea", "pins", "a"); err != nil {
t.Fatal(err)
}
(*made)[0].gone.Store(true)
c.bind = func(context.Context, string) (jetstream.KeyValue, error) {
binds.Add(1)
return nil, jetstream.ErrBucketNotFound
}
if _, err := c.StateGet("gitea", "pins", "a"); err == nil {
t.Fatal("a read of a bucket no longer on the bus answered no error")
}
if got := binds.Load(); got != 2 {
t.Fatalf("the bucket was bound %d times, want twice: the bind again is once", got)
}
})
}
+6 -1
View File
@@ -10,6 +10,8 @@ import (
"time" "time"
"github.com/nats-io/nats.go/jetstream" "github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/process"
) )
// A module's own stream, as the runtime publishes to it and hears it (novox/hq ADR 0363 §8, issue 725). // A module's own stream, as the runtime publishes to it and hears it (novox/hq ADR 0363 §8, issue 725).
@@ -129,7 +131,10 @@ func (h *hearing) bind(stream, durable string, onErr func(error)) (jetstream.Con
if onErr != nil { if onErr != nil {
opts = append(opts, jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { onErr(err) })) opts = append(opts, jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { onErr(err) }))
} }
reading, err := consumer.Consume(h.handle, opts...) // Claimed through the SDK's one guard as it is pulled (design 47 H, H-A-196, H-A-197): a second pull loop on a durable
// this process pulls already would take every other event from the first, so it is refused here, loud. The claim is
// released as the loop is stopped, or once the bus ended it, before its Closed is closed.
reading, err := process.ConsumeOnce(consumer, "the tool runner's hearing of "+h.module, h.handle, opts...)
if err != nil { if err != nil {
return nil, time.Time{}, "reading", err return nil, time.Time{}, "reading", err
} }
@@ -5,6 +5,7 @@ import (
"testing" "testing"
"git.novox.be/novox/mesh-sdk/go/buscalls" "git.novox.be/novox/mesh-sdk/go/buscalls"
"github.com/nats-io/nats.go/jetstream"
) )
// novox/hq ADR 0363 §2, §3: an event goes to the stream whose pattern carries it, as the bus matches subjects. // novox/hq ADR 0363 §2, §3: an event goes to the stream whose pattern carries it, as the bus matches subjects.
@@ -69,3 +70,19 @@ func TestAnEventNoStreamCarriesIsPublishedOnEvents(t *testing.T) {
t.Errorf("a stream's event reaches its consumer as %q, not as on EVENTS", got) t.Errorf("a stream's event reaches its consumer as %q, not as on EVENTS", got)
} }
} }
// Design 47 H lead answers H-A-227: an object store's fill is the figures the bus keeps for its stream, its live
// subjects as the count, never a count of its descriptions: a store holding twenty thousand messages, deleted chunks'
// leftovers among them, answers its bytes, its cap and its four live subjects.
func TestAnObjectStoresFillIsItsStreamsOwnFigures(t *testing.T) {
info := &jetstream.StreamInfo{
Config: jetstream.StreamConfig{MaxBytes: 8 << 30},
State: jetstream.StreamState{Msgs: 20653, Bytes: 1 << 30, NumSubjects: 4, FirstSeq: 9, LastSeq: 20661},
}
if got, want := objectsFillOf(info), (buscalls.Fill{Count: 4, Bytes: 1 << 30, MaxBytes: 8 << 30}); got != want {
t.Fatalf("fill %+v, want %+v", got, want)
}
if got := objectsFillOf(&jetstream.StreamInfo{State: jetstream.StreamState{Bytes: 7}}); got != (buscalls.Fill{Bytes: 7}) {
t.Fatalf("a store with no cap and no live subject: fill %+v", got)
}
}
+28
View File
@@ -258,6 +258,34 @@ func TestAConsumerNamedBeforeItExistsIsBoundOnceItDoes(t *testing.T) {
} }
return nil return nil
}) })
// A second pull loop on the durable the runner pulls is refused, naming it and both binders (design 47 H, H-A-196,
// H-A-197); a re-serve that stops its loop first binds again without a refusal, its module stream's durable too.
if _, err := conn.ConsumeAs("reader", func(bus.Envelope) error { return nil }); err == nil ||
!strings.Contains(err.Error(), "pulled in this process already") || !strings.Contains(err.Error(), "anchor_reader") ||
strings.Count(err.Error(), "the tool runner's hearing of reader") != 2 {
t.Fatalf("a second pull loop on the durable the runner pulls was not refused naming it and both binders: %v", err)
}
stop()
again, err := conn.ConsumeAs("reader", func(env bus.Envelope) error {
mu.Lock()
heard[env.Key]++
mu.Unlock()
return nil
})
if err != nil {
t.Fatalf("the re-serve was refused after its old loop stopped: %v", err)
}
t.Cleanup(again)
mesh.Emit(t, "mesh.stream.mail.event.mail.home.message.added", map[string]any{})
mt.Until(t, func() error {
mu.Lock()
defer mu.Unlock()
if heard["mail.mail.home.message.added"] != 2 {
return fmt.Errorf("after the re-serve heard %v", heard)
}
return nil
})
} }
// novox/hq ADR 0363 §6 (issue 725): a consumer the controller deletes and makes again under the same name, its // novox/hq ADR 0363 §6 (issue 725): a consumer the controller deletes and makes again under the same name, its