Wait out a bus that answers nothing for a few seconds instead of losing the write (issue 774)

At each mesh-grants restart and user-list reload (16:37:24, 16:55:15) nats-server 2.11's reloadAuthorization left
the mesh's account without its imports, $JS.API.> among them, for a few seconds: requests had no responders,
JetStream calls said it was not enabled, and a build-test result on ace was lost.

Quick fix, chosen as asked; the thorough fix is a reload that keeps the account's imports, in the bus's holder.
The package busretry asks a request again, backing off within about 30 s, only while its error says it reached no
one (no responders, JetStream not enabled, no response from the stream, or either in words): nothing was done, so
even a create or an update is safe to ask again. A timeout or any answer is never asked again. After the bound it
is said on standard error and returned. Used by process.State's requests, process.Conn's calls (asked again only
while $JS.API.INFO says the bus itself answers nothing, so a callee that is not running still fails at once) and
asks, and the bundle's state requests through the runtime.

go/process/state.go

@@ -13,6 +13,7 @@ import (
 	"github.com/nats-io/nats.go/jetstream"
 
 	stdio "git.novox.be/novox/mesh-sdk/go"
+	"git.novox.be/novox/mesh-sdk/go/busretry"
 	"git.novox.be/novox/mesh-sdk/go/kvlimit"
 )
 
@@ -74,7 +75,12 @@ func (s State) store(ctx context.Context) (jetstream.KeyValue, error) {
 }
 
 // Get is a key's current value, or nil when it has none.
+// Get waits out a bus that answers nothing for a few seconds (busretry): only a request that reached no one is asked again.
 func (s State) Get(key string) (*stdio.StateEntry, error) {
+	return busretry.Do(s.what("get "+key), func() (*stdio.StateEntry, error) { return s.get(key) }, nil)
+}
+
+func (s State) get(key string) (*stdio.StateEntry, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), stateTimeout)
 	defer cancel()
 	kv, err := s.store(ctx)
@@ -93,7 +99,12 @@ func (s State) Get(key string) (*stdio.StateEntry, error) {
 
 // Create writes a key only when it has none, and returns the revision; a key that has one is refused as
 // stdio.ErrChanged.
+// Create waits out a bus that answers nothing for a few seconds (busretry): only a request that reached no one is asked again.
 func (s State) Create(key string, value any) (uint64, error) {
+	return busretry.Do(s.what("create "+key), func() (uint64, error) { return s.create(key, value) }, nil)
+}
+
+func (s State) create(key string, value any) (uint64, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), stateTimeout)
 	defer cancel()
 	kv, err := s.store(ctx)
@@ -121,7 +132,12 @@ const MinTTL = time.Second
 // 1952 S6): the bus removes it when ttl passes and its watchers hear the removal, so an expiry is an event and no timer
 // runs. A key that has a value is refused as stdio.ErrChanged: an act begun once cannot be begun again. The bucket must
 // take keys with their own time to live (a state declared `key-ttl`).
+// CreateTTL waits out a bus that answers nothing for a few seconds (busretry): only a request that reached no one is asked again.
 func (s State) CreateTTL(key string, value any, ttl time.Duration) (uint64, error) {
+	return busretry.Do(s.what("create "+key), func() (uint64, error) { return s.createTTL(key, value, ttl) }, nil)
+}
+
+func (s State) createTTL(key string, value any, ttl time.Duration) (uint64, error) {
 	if ttl < MinTTL {
 		return 0, fmt.Errorf("%s's state %s, key %s: a time to live of %s is shorter than the bus keeps (%s)", s.Module, s.Name, key, ttl, MinTTL)
 	}
@@ -161,7 +177,12 @@ func (s State) RenewTTL(key string, value any, ttl time.Duration) (uint64, error
 
 // Update writes a key only when it is still at revision, and returns the new one; a key changed meanwhile
 // is refused as stdio.ErrChanged.
+// Update waits out a bus that answers nothing for a few seconds (busretry): only a request that reached no one is asked again.
 func (s State) Update(key string, value any, revision uint64) (uint64, error) {
+	return busretry.Do(s.what("update "+key), func() (uint64, error) { return s.update(key, value, revision) }, nil)
+}
+
+func (s State) update(key string, value any, revision uint64) (uint64, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), stateTimeout)
 	defer cancel()
 	kv, err := s.store(ctx)
@@ -183,7 +204,12 @@ func (s State) Update(key string, value any, revision uint64) (uint64, error) {
 }
 
 // Put writes a key whole, whatever it held, and returns the revision written.
+// Put waits out a bus that answers nothing for a few seconds (busretry): only a request that reached no one is asked again.
 func (s State) Put(key string, value any) (uint64, error) {
+	return busretry.Do(s.what("put "+key), func() (uint64, error) { return s.put(key, value) }, nil)
+}
+
+func (s State) put(key string, value any) (uint64, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), stateTimeout)
 	defer cancel()
 	kv, err := s.store(ctx)
@@ -202,7 +228,12 @@ func (s State) Put(key string, value any) (uint64, error) {
 }
 
 // Keys is every key with a current value, sorted; none is an empty list.
+// Keys waits out a bus that answers nothing for a few seconds (busretry): only a request that reached no one is asked again.
 func (s State) Keys() ([]string, error) {
+	return busretry.Do(s.what("keys"), func() ([]string, error) { return s.keys() }, nil)
+}
+
+func (s State) keys() ([]string, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), stateTimeout)
 	defer cancel()
 	kv, err := s.store(ctx)
@@ -314,7 +345,18 @@ func (s State) Watch(key string, handle func(stdio.StateChange) error) error {
 }
 
 // Delete removes a key. A key that was not there is not an error.
+// Delete waits out a bus that answers nothing for a few seconds, as Put does.
 func (s State) Delete(key string) error {
+	_, err := busretry.Do(s.what("delete "+key), func() (struct{}, error) { return struct{}{}, s.delete(key) }, nil)
+	return err
+}
+
+// what names a request on this state for busretry.
+func (s State) what(request string) string {
+	return fmt.Sprintf("%s's state %s, %s", s.Module, s.Name, request)
+}
+
+func (s State) delete(key string) error {
 	ctx, cancel := context.WithTimeout(context.Background(), stateTimeout)
 	defer cancel()
 	kv, err := s.store(ctx)