Design 47 H1: the tool runner binds a state once per process (H-A-251) #85

Merged
mesh-tools merged 1 commits from fix/47-H1-tools-bind-once into main 2026-11-21 11:01:55 +00:00
5 changed files with 346 additions and 51 deletions
+7
View File
@@ -203,6 +203,13 @@ type Conn struct {
waits map[string]map[chan string]bool
// inboxSaid is when a refused answer publish was last said, by subject (answerRefused).
inboxSaid map[string]time.Time
// bound is each state's binding, kept for the process and bound again only when the bus says its bucket is gone
// (state.go, design 47 H lead answers H-A-251): one Conn is one account's connection, so a binding is per account
// and bucket.
bindMu sync.Mutex
bound map[string]*binding
// bind binds one bucket: jetstream's KeyValue (a stream-info request on the bus) unless a test replaces it.
bind func(ctx context.Context, bucket string) (jetstream.KeyValue, error)
}
// answered is one subject the runtime answers on: what to subscribe again with, and how often it was
+1 -5
View File
@@ -427,11 +427,7 @@ func (c *Conn) stateWrite(module, name, key string, value json.RawMessage,
}
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
defer cancel()
kv, err := c.bucket(ctx, s)
if err != nil {
return 0, err
}
rev, err := write(ctx, kv)
rev, err := inBucket(c, ctx, s, func(kv jetstream.KeyValue) (uint64, error) { return write(ctx, kv) })
if errors.Is(err, jetstream.ErrKeyExists) || isWrongSequence(err) {
return 0, changed{name + "." + key}
}
+116 -46
View File
@@ -15,6 +15,7 @@ import (
"git.novox.be/novox/mesh-sdk/go/membership"
"git.novox.be/novox/mesh-sdk/go/pernode"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
)
@@ -92,20 +93,107 @@ func (c *Conn) issuedState(module, name string) (StateIssued, error) {
"issued: %s (novox/hq ADR 0201, ADR 0595 §1)", module, name, issued)
}
// binding is one bucket's binding, or the bind in flight for it: the first asker binds, an asker meanwhile waits for
// that bind (done) instead of asking the bus again.
type binding struct {
done chan struct{}
kv jetstream.KeyValue
err error
}
// bucket is the state's binding, bound once for the process (design 47 H lead answers H-A-251): binding a bucket is a
// stream-info request on the bus, and asked before every get it made each read two answers. A bind that fails is not
// kept, so the next ask binds again.
func (c *Conn) bucket(ctx context.Context, s StateIssued) (jetstream.KeyValue, error) {
c.bindMu.Lock()
b := c.bound[s.Bucket]
if b == nil {
if c.bound == nil {
c.bound = map[string]*binding{}
}
b = &binding{done: make(chan struct{})}
c.bound[s.Bucket] = b
c.bindMu.Unlock()
b.kv, b.err = c.bindBucket(ctx, s.Bucket)
if b.err != nil {
c.bindMu.Lock()
if c.bound[s.Bucket] == b {
delete(c.bound, s.Bucket)
}
c.bindMu.Unlock()
}
close(b.done)
} else {
c.bindMu.Unlock()
select {
case <-b.done:
case <-ctx.Done():
return nil, ctx.Err()
}
}
if b.err != nil {
if errors.Is(b.err, jetstream.ErrBucketNotFound) {
return nil, fmt.Errorf("the state %q is issued and its bucket is not on the bus yet: the mesh "+
"creates it from the catalogue on its next raise", s.Name)
}
return nil, b.err
}
return b.kv, nil
}
func (c *Conn) bindBucket(ctx context.Context, bucket string) (jetstream.KeyValue, error) {
if c.bind != nil {
return c.bind(ctx, bucket)
}
js, err := jetstream.New(c.nc)
if err != nil {
return nil, err
}
kv, err := js.KeyValue(ctx, s.Bucket)
if err != nil {
if errors.Is(err, jetstream.ErrBucketNotFound) {
return nil, fmt.Errorf("the state %q is issued and its bucket is not on the bus yet: the mesh "+
"creates it from the catalogue on its next raise", s.Name)
}
return nil, err
return js.KeyValue(ctx, bucket)
}
// unbind drops a bucket's binding the bus said is gone, when it is still the one kept, so the next ask binds again.
func (c *Conn) unbind(bucket string, kv jetstream.KeyValue) {
c.bindMu.Lock()
defer c.bindMu.Unlock()
b := c.bound[bucket]
if b == nil {
return
}
return kv, nil
select {
case <-b.done: // bound: its kv is written
default:
return // bound again already, in flight
}
if b.kv == kv {
delete(c.bound, bucket)
}
}
// bucketGone is the bus saying a kept binding's bucket is not there (deleted, or deleted and created again): no stream of
// the name, or nobody answers on its subjects.
func bucketGone(err error) bool {
return errors.Is(err, jetstream.ErrStreamNotFound) || errors.Is(err, jetstream.ErrBucketNotFound) ||
errors.Is(err, jetstream.ErrNoStreamResponse) || errors.Is(err, nats.ErrNoResponders)
}
// inBucket does one act on the state's kept binding; when the bus says its bucket is gone, the binding is dropped, the
// bucket bound again once and the act done once more on the new binding (H-A-251).
func inBucket[T any](c *Conn, ctx context.Context, s StateIssued, do func(jetstream.KeyValue) (T, error)) (T, error) {
var none T
kv, err := c.bucket(ctx, s)
if err != nil {
return none, err
}
got, err := do(kv)
if !bucketGone(err) {
return got, err
}
c.unbind(s.Bucket, kv)
if kv, err = c.bucket(ctx, s); err != nil {
return none, err
}
return do(kv)
}
func checkKey(key string) error {
@@ -130,11 +218,7 @@ func (c *Conn) StateGet(module, name, key string) (*StateEntry, error) {
}
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
defer cancel()
kv, err := c.bucket(ctx, s)
if err != nil {
return nil, err
}
e, err := kv.Get(ctx, key)
e, err := inBucket(c, ctx, s, func(kv jetstream.KeyValue) (jetstream.KeyValueEntry, error) { return kv.Get(ctx, key) })
if errors.Is(err, jetstream.ErrKeyNotFound) {
return nil, nil
}
@@ -210,11 +294,10 @@ func (c *Conn) StatePage(module, name, after string, maxBytes int, budget time.D
maps.Copy(current, got)
}
} else {
kv, err := c.bucket(ctx, s)
if err != nil {
return StatePage{}, err
}
if current, err = currentOf(ctx, kv); err != nil {
var err error
if current, err = inBucket(c, ctx, s, func(kv jetstream.KeyValue) (map[string]*StateEntry, error) {
return currentOf(ctx, kv)
}); err != nil {
return StatePage{}, err
}
}
@@ -273,11 +356,9 @@ func (c *Conn) StateHistory(module, name, key string) ([]StateKept, error) {
}
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
defer cancel()
kv, err := c.bucket(ctx, s)
if err != nil {
return nil, err
}
entries, err := kv.History(ctx, key)
entries, err := inBucket(c, ctx, s, func(kv jetstream.KeyValue) ([]jetstream.KeyValueEntry, error) {
return kv.History(ctx, key)
})
kept := []StateKept{}
if errors.Is(err, jetstream.ErrKeyNotFound) {
return kept, nil
@@ -326,11 +407,7 @@ func (c *Conn) StateInfo(module, name string) (*StateInfo, error) {
}
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
defer cancel()
kv, err := c.bucket(ctx, s)
if err != nil {
return nil, err
}
status, err := kv.Status(ctx)
status, err := inBucket(c, ctx, s, func(kv jetstream.KeyValue) (jetstream.KeyValueStatus, error) { return kv.Status(ctx) })
if err != nil {
return nil, err
}
@@ -363,11 +440,7 @@ func (c *Conn) StatePut(module, name, key string, value json.RawMessage) (uint64
}
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
defer cancel()
kv, err := c.bucket(ctx, s)
if err != nil {
return 0, err
}
return kv.Put(ctx, key, value)
return inBucket(c, ctx, s, func(kv jetstream.KeyValue) (uint64, error) { return kv.Put(ctx, key, value) })
}
// MinKeyTTL is the shortest time to live a key may be given: the bus counts a key's age in whole seconds (mesh-sdk
@@ -437,11 +510,8 @@ func (c *Conn) StateDelete(module, name, key string) error {
}
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
defer cancel()
kv, err := c.bucket(ctx, s)
if err != nil {
return err
}
return kv.Delete(ctx, key)
_, err = inBucket(c, ctx, s, func(kv jetstream.KeyValue) (struct{}, error) { return struct{}{}, kv.Delete(ctx, key) })
return err
}
// StateKeys is every key with a current value, sorted.
@@ -455,11 +525,7 @@ func (c *Conn) StateKeys(module, name string) ([]string, error) {
}
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
defer cancel()
kv, err := c.bucket(ctx, s)
if err != nil {
return nil, err
}
return keysOf(ctx, kv)
return inBucket(c, ctx, s, func(kv jetstream.KeyValue) ([]string, error) { return keysOf(ctx, kv) })
}
// keysOf is every key with a current value in one bucket, sorted.
@@ -529,12 +595,16 @@ func (c *Conn) StateWatch(module, name, pattern string, deliver func(StateChange
return c.perNodeWatch(module, s, pattern, deliver)
}
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
kv, err := c.bucket(ctx, s)
cancel()
defer cancel()
type watching struct{ stop func() }
w, err := inBucket(c, ctx, s, func(kv jetstream.KeyValue) (watching, error) {
stop, err := c.watchBucket(module, name, kv, filter, deliver)
return watching{stop}, err
})
if err != nil {
return nil, err
}
return c.watchBucket(module, name, kv, filter, deliver)
return w.stop, nil
}
// watchBucket is StateWatch over one bucket: the current values, then every change.
@@ -0,0 +1,81 @@
//go:build integration
package bus_test
import (
"strings"
"sync/atomic"
"testing"
"github.com/nats-io/nats.go"
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
)
// **On the bus, a read of a state is one answer: its bucket is bound once, and bound again when it was made afresh or
// is gone** (design 47 H lead answers H-A-251). The runtime asked the bucket's stream info before every get, so each
// read was two answers; a third of a night's bus flood came this way.
func TestAStateReadOnTheBusBindsItsBucketOnce(t *testing.T) {
conn, kv := owedWatcher(t)
const bucket = "mesh-delivery-assigner_pins"
if _, err := kv.Put("a", []byte(`{"v":1}`)); err != nil {
t.Fatal(err)
}
watcher, err := nats.Connect(mt.URL(t))
if err != nil {
t.Fatal(err)
}
t.Cleanup(watcher.Close)
var infos atomic.Int32
if _, err := watcher.Subscribe("$JS.API.STREAM.INFO.KV_"+bucket, func(*nats.Msg) { infos.Add(1) }); err != nil {
t.Fatal(err)
}
if err := watcher.Flush(); err != nil {
t.Fatal(err)
}
read := func() {
t.Helper()
e, err := conn.StateGet("gitea", "mesh-delivery-assigner.pins", "a")
if err != nil || e == nil {
t.Fatalf("read: %+v, %v", e, err)
}
}
read()
read()
read()
if err := watcher.Flush(); err != nil {
t.Fatal(err)
}
if got := infos.Load(); got != 1 {
t.Fatalf("three reads asked the bucket's stream info %d times, want once", got)
}
// The bucket deleted under the runtime: the read binds again, once, and says the bucket is not on the bus.
js, _ := watcher.JetStream()
if err := js.DeleteKeyValue(bucket); err != nil {
t.Fatal(err)
}
_, err = conn.StateGet("gitea", "mesh-delivery-assigner.pins", "a")
if err == nil || !strings.Contains(err.Error(), "not on the bus yet") {
t.Fatalf("a read of a deleted bucket answered %v, want that its bucket is not on the bus", err)
}
// And made afresh: read again from the new bucket.
again, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: bucket, History: 1})
if err != nil {
t.Fatal(err)
}
if _, err := again.Put("a", []byte(`{"v":2}`)); err != nil {
t.Fatal(err)
}
e, err := conn.StateGet("gitea", "mesh-delivery-assigner.pins", "a")
if err != nil || e == nil || string(e.Value) != `{"v":2}` {
t.Fatalf("a read of the bucket made afresh: %+v, %v", e, err)
}
read()
if err := watcher.Flush(); err != nil {
t.Fatal(err)
}
if got := infos.Load(); got != 3 {
t.Fatalf("the bucket's stream info was asked %d times in all, want 3: once, once more when it was gone, once when made afresh", got)
}
}
+141
View File
@@ -0,0 +1,141 @@
package bus
import (
"context"
"encoding/json"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/nats-io/nats.go/jetstream"
)
// boundKV is a bucket's binding in a test: it answers every get with one value, or, once gone, as the bus answers a
// binding whose bucket was deleted.
type boundKV struct {
jetstream.KeyValue
gone atomic.Bool
puts atomic.Int32
}
type boundEntry struct {
jetstream.KeyValueEntry
key string
}
func (e boundEntry) Key() string { return e.key }
func (e boundEntry) Value() []byte { return []byte(`{"v":1}`) }
func (e boundEntry) Revision() uint64 { return 1 }
func (boundEntry) Created() time.Time { return time.Time{} }
func (*boundKV) Bucket() string { return "gitea_pins" }
func (kv *boundKV) Get(_ context.Context, key string) (jetstream.KeyValueEntry, error) {
if kv.gone.Load() {
return nil, jetstream.ErrStreamNotFound
}
return boundEntry{key: key}, nil
}
func (kv *boundKV) Put(context.Context, string, []byte) (uint64, error) {
if kv.gone.Load() {
return 0, jetstream.ErrNoStreamResponse
}
return uint64(kv.puts.Add(1)), nil
}
// **A state is bound once for the process, and bound again only when the bus says its bucket is gone** (design 47 H
// lead answers H-A-251): binding is a stream-info request on the bus, and asked before every get it made each read two
// answers on the bus.
func TestAStateIsBoundOnceAndBoundAgainWhenItsBucketIsGone(t *testing.T) {
setup := func() (*Conn, *atomic.Int32, *[]*boundKV) {
var binds atomic.Int32
var mu sync.Mutex
made := []*boundKV{}
c := &Conn{node: "anchor", issued: map[string]*Membership{"gitea": {Module: "gitea",
State: []StateIssued{{Name: "pins", Bucket: "gitea_pins", Writes: true}}}}}
c.bind = func(_ context.Context, bucket string) (jetstream.KeyValue, error) {
binds.Add(1)
if bucket != "gitea_pins" {
t.Errorf("bound %s, want gitea_pins", bucket)
}
kv := &boundKV{}
mu.Lock()
made = append(made, kv)
mu.Unlock()
return kv, nil
}
return c, &binds, &made
}
t.Run("two reads, one bind", func(t *testing.T) {
c, binds, _ := setup()
for range 2 {
if e, err := c.StateGet("gitea", "pins", "a"); err != nil || e == nil || e.Key != "a" {
t.Fatalf("read: %+v, %v", e, err)
}
}
if _, err := c.StatePut("gitea", "pins", "a", json.RawMessage(`{"v":2}`)); err != nil {
t.Fatal(err)
}
if got := binds.Load(); got != 1 {
t.Fatalf("two reads and a write bound the bucket %d times, want once", got)
}
})
t.Run("many readers at once, one bind", func(t *testing.T) {
c, binds, _ := setup()
var wg sync.WaitGroup
for range 16 {
wg.Go(func() {
if _, err := c.StateGet("gitea", "pins", "a"); err != nil {
t.Error(err)
}
})
}
wg.Wait()
if got := binds.Load(); got != 1 {
t.Fatalf("16 readers bound the bucket %d times, want once", got)
}
})
t.Run("a bucket gone under the runner is bound again once", func(t *testing.T) {
c, binds, made := setup()
if _, err := c.StateGet("gitea", "pins", "a"); err != nil {
t.Fatal(err)
}
(*made)[0].gone.Store(true)
if e, err := c.StateGet("gitea", "pins", "a"); err != nil || e == nil {
t.Fatalf("the read after the bucket was made again: %+v, %v", e, err)
}
if got := binds.Load(); got != 2 {
t.Fatalf("the bucket was bound %d times, want twice (once more after it was gone)", got)
}
if _, err := c.StateGet("gitea", "pins", "a"); err != nil {
t.Fatal(err)
}
(*made)[1].gone.Store(true)
if _, err := c.StatePut("gitea", "pins", "a", json.RawMessage(`{"v":2}`)); err != nil {
t.Fatal(err)
}
if got, puts := binds.Load(), (*made)[2].puts.Load(); got != 3 || puts != 1 {
t.Fatalf("a write to a gone bucket bound it %d times in all (want 3) and wrote %d times on the new binding (want 1)", got, puts)
}
})
t.Run("a bucket still gone after the bind again is said", func(t *testing.T) {
c, binds, made := setup()
if _, err := c.StateGet("gitea", "pins", "a"); err != nil {
t.Fatal(err)
}
(*made)[0].gone.Store(true)
c.bind = func(context.Context, string) (jetstream.KeyValue, error) {
binds.Add(1)
return nil, jetstream.ErrBucketNotFound
}
if _, err := c.StateGet("gitea", "pins", "a"); err == nil {
t.Fatal("a read of a bucket no longer on the bus answered no error")
}
if got := binds.Load(); got != 2 {
t.Fatalf("the bucket was bound %d times, want twice: the bind again is once", got)
}
})
}