go/busretry/busretry.go

@@ -0,0 +1,69 @@
+// Package busretry waits out a bus that answers nothing for a few seconds (issue 774, the sweep of 2026-10-30: bus
+// blackouts at every user-list reload). nats-server 2.11's reloadAuthorization rebuilds the accounts, and for a few
+// seconds the mesh's account has none of its imports, $JS.API.> among them: every request is answered "no responders"
+// and every JetStream call "jetstream not enabled". A write or a call made in that window failed, and was lost.
+//
+// Only an error that says the request reached no one is waited out (Undelivered): nothing was done, so asking again
+// is safe even for a write that is not idempotent. A timeout, or any answer, is never asked again: the request may
+// have been delivered. The wait is bounded (Bound, about 30 s, backing off), and said on standard error when it ends
+// without an answer.
+//
+// This is the quick fix. The thorough one is a reload that keeps the account's imports, in the bus's holder.
+package busretry
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// Bound is how long a request that reached no one is asked again; FirstWait the first pause, doubled to MostWait.
+var (
+	Bound     = 30 * time.Second
+	FirstWait = 250 * time.Millisecond
+	MostWait  = 4 * time.Second
+)
+
+// Undelivered says err means the request reached no one: no responders, JetStream not enabled (for the account), or
+// no response from the stream after the client's own retries on no responders; also said in words by a runtime that
+// passed the error on.
+func Undelivered(err error) bool {
+	if err == nil {
+		return false
+	}
+	if errors.Is(err, nats.ErrNoResponders) || errors.Is(err, jetstream.ErrJetStreamNotEnabled) ||
+		errors.Is(err, jetstream.ErrJetStreamNotEnabledForAccount) || errors.Is(err, jetstream.ErrNoStreamResponse) ||
+		errors.Is(err, nats.ErrJetStreamNotEnabled) || errors.Is(err, nats.ErrJetStreamNotEnabledForAccount) {
+		return true
+	}
+	words := strings.ToLower(err.Error())
+	return strings.Contains(words, "no responders") || strings.Contains(words, "jetstream not enabled")
+}
+
+// Do runs try, and again while its error says the request reached no one, until Bound has passed. blackout, when
+// given, is asked after each such error and Do stops waiting when it says the bus is not blacked out: a call nobody
+// answers because its callee is not running fails at once, as before. what names the request in the error.
+func Do[T any](what string, try func() (T, error), blackout func() bool) (T, error) {
+	v, err := try()
+	if !Undelivered(err) || (blackout != nil && !blackout()) {
+		return v, err
+	}
+	start, wait := time.Now(), FirstWait
+	for time.Since(start) < Bound {
+		time.Sleep(wait)
+		wait = min(wait*2, MostWait)
+		v, err = try()
+		if !Undelivered(err) || (blackout != nil && !blackout()) {
+			return v, err
+		}
+	}
+	err = fmt.Errorf("%s: the bus answered nothing for %s, asked again until then (a user-list reload drops the "+
+		"account's imports for a few seconds; this was longer): %w", what, time.Since(start).Round(time.Second), err)
+	fmt.Fprintf(os.Stderr, "[mesh-sdk] %v\n", err)
+	return v, err
+}

go/busretry/busretry_test.go

@@ -0,0 +1,48 @@
+package busretry
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// Issue 774 (bus blackouts at every user-list reload): only a request that reached no one is asked again, within the
+// bound; a blackout probe that says the bus answers ends the wait at once.
+func TestABusThatAnswersNothingIsWaitedOut(t *testing.T) {
+	Bound, FirstWait, MostWait = 60*time.Millisecond, time.Millisecond, 4*time.Millisecond
+	for _, row := range []struct {
+		name      string
+		errs      []error // what each try answers; the last repeats
+		blackout  func() bool
+		tries     int // at least
+		atMost    int
+		delivered bool
+	}{
+		{"no responders, then an answer", []error{nats.ErrNoResponders, nats.ErrNoResponders, nil}, nil, 3, 3, true},
+		{"jetstream not enabled, then an answer", []error{jetstream.ErrJetStreamNotEnabled, nil}, nil, 2, 2, true},
+		{"said in words by a runtime", []error{errors.New("nats: no responders available for request"), nil}, nil, 2, 2, true},
+		{"a timeout may have been delivered: never asked again", []error{context.DeadlineExceeded}, nil, 1, 1, false},
+		{"an answer that refuses is an answer", []error{errors.New("refused")}, nil, 1, 1, false},
+		{"nobody answers on a bus that answers: the callee is not running", []error{nats.ErrNoResponders}, func() bool { return false }, 1, 1, false},
+		{"blacked out past the bound: said", []error{jetstream.ErrJetStreamNotEnabledForAccount}, func() bool { return true }, 3, 1000, false},
+	} {
+		tries := 0
+		_, err := Do("the test", func() (int, error) {
+			e := row.errs[min(tries, len(row.errs)-1)]
+			tries++
+			return tries, e
+		}, row.blackout)
+		if tries < row.tries || tries > row.atMost || (err == nil) != row.delivered {
+			t.Errorf("%s: %d tries, %v", row.name, tries, err)
+		}
+		if strings.HasPrefix(row.name, "blacked out") && (!Undelivered(err) || !strings.Contains(fmt.Sprint(err), "the bus answered nothing")) {
+			t.Errorf("%s: %v", row.name, err)
+		}
+	}
+}

go/process/asks.go

@@ -6,7 +6,10 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/nats-io/nats.go"
+
 	stdio "git.novox.be/novox/mesh-sdk/go"
+	"git.novox.be/novox/mesh-sdk/go/busretry"
 )
 
 // Ask reads the router's record of one of this process's module's own asks, under its own module and no
@@ -24,9 +27,12 @@ func (c *Conn) Ask(bucket, id string) (*stdio.StateEntry, error) {
 		return nil, err
 	}
 	subject := "$JS.API.DIRECT.GET.KV_" + bucket + ".$KV." + bucket + "." + c.account.Module + "." + id
-	ctx, cancel := context.WithTimeout(context.Background(), stateTimeout)
-	defer cancel()
-	reply, err := c.nc.RequestWithContext(ctx, subject, nil)
+	// A read: asked again while the bus answers nothing (busretry).
+	reply, err := busretry.Do("reading "+subject, func() (*nats.Msg, error) {
+		ctx, cancel := context.WithTimeout(context.Background(), stateTimeout)
+		defer cancel()
+		return c.nc.RequestWithContext(ctx, subject, nil)
+	}, nil)
 	if err != nil {
 		return nil, fmt.Errorf("reading %s's ask %s from %s: %w", c.account.Module, id, bucket, err)
 	}

go/process/call.go

@@ -9,6 +9,8 @@ import (
 	"time"
 
 	"github.com/nats-io/nats.go"
+
+	"git.novox.be/novox/mesh-sdk/go/busretry"
 )
 
 // CallTimeout bounds a call to a tool or a seat verb (mesh-tools node-tools/internal/bus/bus.go:35,
@@ -47,16 +49,34 @@ func (c *Conn) CallOn(node, key string, body any) (json.RawMessage, error) {
 	return c.callOn(called, fallback, body)
 }
 
+// callOn asks called, then fallback when nothing answers it; while the bus itself answers nothing (a user-list reload
+// drops the account's imports for a few seconds, busretry) both are asked again, within busretry.Bound. A call nobody
+// answers on a bus that answers is not asked again: its callee is not running. A call that may have been delivered (a
+// timeout, any answer) is never asked again.
 func (c *Conn) callOn(called, fallback string, body any) (json.RawMessage, error) {
 	data, err := json.Marshal(body)
 	if err != nil {
 		return nil, err
 	}
-	result, err := c.request(called, data)
-	if errors.Is(err, nats.ErrNoResponders) {
-		result, err = c.request(fallback, data)
+	return busretry.Do("the call on "+fallback, func() (json.RawMessage, error) {
+		result, err := c.request(called, data)
+		if errors.Is(err, nats.ErrNoResponders) {
+			result, err = c.request(fallback, data)
+		}
+		return result, err
+	}, c.blackedOut)
+}
+
+// blackedOut says the bus answers this account nothing at all: its JetStream API, which every account imports, has no
+// responder or says it is not enabled.
+func (c *Conn) blackedOut() bool {
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
+	msg, err := c.nc.RequestWithContext(ctx, "$JS.API.INFO", nil)
+	if err != nil {
+		return busretry.Undelivered(err)
 	}
-	return result, err
+	return strings.Contains(string(msg.Data), `"err_code":10039`) || strings.Contains(string(msg.Data), "jetstream not enabled")
 }
 
 // callerToken is this account's user name as the last token of a call it publishes, each dot written ~

go/process/state.go

@@ -13,6 +13,7 @@ import (
 	"github.com/nats-io/nats.go/jetstream"
 
 	stdio "git.novox.be/novox/mesh-sdk/go"
+	"git.novox.be/novox/mesh-sdk/go/busretry"
 	"git.novox.be/novox/mesh-sdk/go/kvlimit"
 )
 
@@ -74,7 +75,12 @@ func (s State) store(ctx context.Context) (jetstream.KeyValue, error) {
 }
 
 // Get is a key's current value, or nil when it has none.
+// Get waits out a bus that answers nothing for a few seconds (busretry): only a request that reached no one is asked again.
 func (s State) Get(key string) (*stdio.StateEntry, error) {
+	return busretry.Do(s.what("get "+key), func() (*stdio.StateEntry, error) { return s.get(key) }, nil)
+}
+
+func (s State) get(key string) (*stdio.StateEntry, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), stateTimeout)
 	defer cancel()
 	kv, err := s.store(ctx)
@@ -93,7 +99,12 @@ func (s State) Get(key string) (*stdio.StateEntry, error) {
 
 // Create writes a key only when it has none, and returns the revision; a key that has one is refused as
 // stdio.ErrChanged.
+// Create waits out a bus that answers nothing for a few seconds (busretry): only a request that reached no one is asked again.
 func (s State) Create(key string, value any) (uint64, error) {
+	return busretry.Do(s.what("create "+key), func() (uint64, error) { return s.create(key, value) }, nil)
+}
+
+func (s State) create(key string, value any) (uint64, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), stateTimeout)
 	defer cancel()
 	kv, err := s.store(ctx)
@@ -121,7 +132,12 @@ const MinTTL = time.Second
 // 1952 S6): the bus removes it when ttl passes and its watchers hear the removal, so an expiry is an event and no timer
 // runs. A key that has a value is refused as stdio.ErrChanged: an act begun once cannot be begun again. The bucket must
 // take keys with their own time to live (a state declared `key-ttl`).
+// CreateTTL waits out a bus that answers nothing for a few seconds (busretry): only a request that reached no one is asked again.
 func (s State) CreateTTL(key string, value any, ttl time.Duration) (uint64, error) {
+	return busretry.Do(s.what("create "+key), func() (uint64, error) { return s.createTTL(key, value, ttl) }, nil)
+}
+
+func (s State) createTTL(key string, value any, ttl time.Duration) (uint64, error) {
 	if ttl < MinTTL {
 		return 0, fmt.Errorf("%s's state %s, key %s: a time to live of %s is shorter than the bus keeps (%s)", s.Module, s.Name, key, ttl, MinTTL)
 	}
@@ -161,7 +177,12 @@ func (s State) RenewTTL(key string, value any, ttl time.Duration) (uint64, error
 
 // Update writes a key only when it is still at revision, and returns the new one; a key changed meanwhile
 // is refused as stdio.ErrChanged.
+// Update waits out a bus that answers nothing for a few seconds (busretry): only a request that reached no one is asked again.
 func (s State) Update(key string, value any, revision uint64) (uint64, error) {
+	return busretry.Do(s.what("update "+key), func() (uint64, error) { return s.update(key, value, revision) }, nil)
+}
+
+func (s State) update(key string, value any, revision uint64) (uint64, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), stateTimeout)
 	defer cancel()
 	kv, err := s.store(ctx)
@@ -183,7 +204,12 @@ func (s State) Update(key string, value any, revision uint64) (uint64, error) {
 }
 
 // Put writes a key whole, whatever it held, and returns the revision written.
+// Put waits out a bus that answers nothing for a few seconds (busretry): only a request that reached no one is asked again.
 func (s State) Put(key string, value any) (uint64, error) {
+	return busretry.Do(s.what("put "+key), func() (uint64, error) { return s.put(key, value) }, nil)
+}
+
+func (s State) put(key string, value any) (uint64, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), stateTimeout)
 	defer cancel()
 	kv, err := s.store(ctx)
@@ -202,7 +228,12 @@ func (s State) Put(key string, value any) (uint64, error) {
 }
 
 // Keys is every key with a current value, sorted; none is an empty list.
+// Keys waits out a bus that answers nothing for a few seconds (busretry): only a request that reached no one is asked again.
 func (s State) Keys() ([]string, error) {
+	return busretry.Do(s.what("keys"), func() ([]string, error) { return s.keys() }, nil)
+}
+
+func (s State) keys() ([]string, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), stateTimeout)
 	defer cancel()
 	kv, err := s.store(ctx)
@@ -314,7 +345,18 @@ func (s State) Watch(key string, handle func(stdio.StateChange) error) error {
 }
 
 // Delete removes a key. A key that was not there is not an error.
+// Delete waits out a bus that answers nothing for a few seconds, as Put does.
 func (s State) Delete(key string) error {
+	_, err := busretry.Do(s.what("delete "+key), func() (struct{}, error) { return struct{}{}, s.delete(key) }, nil)
+	return err
+}
+
+// what names a request on this state for busretry.
+func (s State) what(request string) string {
+	return fmt.Sprintf("%s's state %s, %s", s.Module, s.Name, request)
+}
+
+func (s State) delete(key string) error {
 	ctx, cancel := context.WithTimeout(context.Background(), stateTimeout)
 	defer cancel()
 	kv, err := s.store(ctx)

go/process/state_test.go

@@ -13,6 +13,7 @@ import (
 
 	stdio "git.novox.be/novox/mesh-sdk/go"
 	"git.novox.be/novox/mesh-sdk/go/asker"
+	"git.novox.be/novox/mesh-sdk/go/busretry"
 )
 
 // fakeEntry is a fakeKV's KeyValueEntry: only Key, Value and Revision are read by State.
@@ -206,6 +207,19 @@ func TestStatePutWritesAKeyWhole(t *testing.T) {
 	if err != nil || e == nil || e.Revision != rev2 || string(e.Value) != `{"a":2}` {
 		t.Fatalf("Get after Put: %+v, %v", e, err)
 	}
+	// Issue 774: a bus that answers nothing for a moment (a user-list reload) is waited out, not a lost write.
+	busretry.FirstWait = time.Millisecond
+	kv, blackedOut := newFakeKV(), 2
+	dark := State{Module: "mesh-mail", Name: "sends", kv: func(context.Context) (jetstream.KeyValue, error) {
+		if blackedOut > 0 {
+			blackedOut--
+			return nil, jetstream.ErrJetStreamNotEnabled
+		}
+		return kv, nil
+	}}
+	if rev, err := dark.Put("k", 1); err != nil || rev == 0 || blackedOut != 0 {
+		t.Fatalf("a Put through a short blackout: %d, %v", rev, err)
+	}
 }
 
 // 1840/S1: given a state, when its keys are listed, then they come sorted, and none is an empty list, not an error.

go/state.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"time"
 
+	"git.novox.be/novox/mesh-sdk/go/busretry"
 	"git.novox.be/novox/mesh-sdk/go/kvlimit"
 )
 
@@ -85,7 +86,10 @@ func (s KeptState) ask(verb string, params map[string]any) (json.RawMessage, err
 		return nil, err
 	}
 	params["state"] = s.name
-	raw, err := c.ask("mesh/state."+verb, params, stateTimeout)
+	// A request the runtime's bus delivered to no one is asked again for a few seconds (busretry): nothing was done.
+	raw, err := busretry.Do("the state "+s.name+", "+verb+" "+key, func() (json.RawMessage, error) {
+		return c.ask("mesh/state."+verb, params, stateTimeout)
+	}, nil)
 	if size >= 0 {
 		err = kvlimit.Refused("the state "+s.name, key, size, kvlimit.ModuleStateValueBytes, err)
 	}