The tool runner reads the memberships state again until it can, instead of giving a module up at its one follow

A JetStream 503 or a timeout at the follow (a bus right after its boot) left the module with no membership for good, since
Follow runs once per module and the stream fallback is gone. The read is now tried again with a growing wait (1 s doubling
to 30 s) until it watches the key or the connection closes; the failure is said once, and its reading once it succeeds.
Tests: none added; a step added to an existing unit test (TestAModuleHearsEveryStreamItsMembershipNames), which fails
without the retry.

(issue 774, 774/S32, review g1 round 1)
Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL

node-tools/internal/bus/bus.go

@@ -366,6 +366,13 @@ func (c *Conn) Follow(module string) {
 // GrantsPatience bounds the first read of a membership from the grants part's state.
 var GrantsPatience = 2 * time.Second
 
+// GrantsRetry is the first wait before the memberships state is read again after it could not be; it doubles up to
+// GrantsRetryMost.
+var (
+	GrantsRetry     = time.Second
+	GrantsRetryMost = 30 * time.Second
+)
+
 // issue applies a membership that arrived for module, live, and re-serves on it.
 func (c *Conn) issue(module string, m Membership) {
 	c.mu.Lock()
@@ -383,9 +390,38 @@ func (c *Conn) issue(module string, m Membership) {
 }
 
 // followGrants reads module's key of the grants part's state memberships and watches it: the value there now is applied
-// before Follow returns, each change after it live. A state that is not there yet, or not readable, is said once, and
-// the module has no membership until it can be read.
+// before Follow returns, each change after it live. A state that is not there yet, or not readable (a bus answering 503
+// right after its boot, a timeout), is said once and read again, with a growing wait, until it can be or the
+// connection closes: Follow runs once per module, so nothing else would ever read it again.
 func (c *Conn) followGrants(module string) {
+	if c.watchGrants(module, true) {
+		return
+	}
+	go func() {
+		wait := GrantsRetry
+		for !c.nc.IsClosed() {
+			time.Sleep(wait)
+			if c.nc.IsClosed() {
+				return
+			}
+			if c.watchGrants(module, false) {
+				c.Logf("[mesh-tools] the grants part's memberships are read again; %s on %s follows its key", module, c.node)
+				return
+			}
+			if wait *= 2; wait > GrantsRetryMost {
+				wait = GrantsRetryMost
+			}
+		}
+	}()
+}
+
+// watchGrants is one attempt of followGrants: true once the key is watched. A failure is said when say is true.
+func (c *Conn) watchGrants(module string, say bool) bool {
+	logf := func(format string, a ...any) {
+		if say {
+			c.Logf(format, a...)
+		}
+	}
 	key := buscompose.MembershipKey(c.node, module)
 	// First the key's direct get, which the account is granted alone: a bus that refuses it, or a state not there yet,
 	// answers at once, where binding the bucket would wait out its bound on a refusal.
@@ -396,27 +432,27 @@ func (c *Conn) followGrants(module string) {
 		if err != nil {
 			why = err.Error()
 		}
-		c.Logf("[mesh-tools] the grants part's memberships (%s) cannot be read (%s); %s has no membership "+
+		logf("[mesh-tools] the grants part's memberships (%s) cannot be read (%s); %s has no membership "+
 			"until it can be", bucket, why, module)
-		return
+		return false
 	}
 	js, err := jetstream.New(c.nc)
 	if err != nil {
-		return
+		return false
 	}
 	ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
 	defer cancel()
 	kv, err := js.KeyValue(ctx, bucket)
 	if err != nil {
-		c.Logf("[mesh-tools] the grants part's memberships (%s) cannot be read (%v); %s has no membership "+
+		logf("[mesh-tools] the grants part's memberships (%s) cannot be read (%v); %s has no membership "+
 			"until it can be", buscompose.GrantsMembershipsBucket, err, module)
-		return
+		return false
 	}
 	w, err := kv.Watch(context.Background(), key)
 	if err != nil {
-		c.Logf("[mesh-tools] %s's key %s of the grants part's memberships cannot be watched (%v); %s has no membership "+
+		logf("[mesh-tools] %s's key %s of the grants part's memberships cannot be watched (%v); %s has no membership "+
 			"until it can be", module, key, err, module)
-		return
+		return false
 	}
 	apply := func(e jetstream.KeyValueEntry) {
 		if e.Operation() != jetstream.KeyValuePut {
@@ -443,7 +479,7 @@ func (c *Conn) followGrants(module string) {
 		select {
 		case e, ok := <-w.Updates():
 			if !ok {
-				return
+				return false
 			}
 			if e == nil {
 				current = false
@@ -452,9 +488,9 @@ func (c *Conn) followGrants(module string) {
 			apply(e)
 		case <-timer.C:
 			_ = w.Stop()
-			c.Logf("[mesh-tools] %s's key %s of the grants part's memberships did not answer in %s; %s has no membership "+
+			logf("[mesh-tools] %s's key %s of the grants part's memberships did not answer in %s; %s has no membership "+
 				"until it can be", module, key, StateTimeout, module)
-			return
+			return false
 		}
 	}
 	go func() {
@@ -464,6 +500,7 @@ func (c *Conn) followGrants(module string) {
 			}
 		}
 	}()
+	return true
 }
 
 // Membership is what the mesh issued a module here, or nil when nothing has been issued.

node-tools/internal/bus/streams_test.go

@@ -199,6 +199,29 @@ func TestAModuleHearsEveryStreamItsMembershipNames(t *testing.T) {
 	if got := consumerOf(); got != "from-grants-2" {
 		t.Fatalf("a membership said on the controller's subject was applied: the membership names %q", got)
 	}
+	// Review g1 round 1: memberships that cannot be read at the follow (a bus not ready after its boot) are read again until
+	// they can be; the follow, which runs once per module, does not give the module up for good.
+	was := bus.GrantsRetry
+	bus.GrantsRetry = 50 * time.Millisecond
+	t.Cleanup(func() { bus.GrantsRetry = was })
+	mesh.DropGrants(t)
+	late, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: "node-tools", Node: "anchor"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	late.Logf = func(string, ...any) {}
+	t.Cleanup(late.Close)
+	late.Follow("reader")
+	if m := late.Membership("reader"); m != nil {
+		t.Fatalf("a membership was served from a state not there: %+v", m)
+	}
+	mesh.Issue(t, granted)
+	mt.Until(t, func() error {
+		if m := late.Membership("reader"); m == nil || len(m.Hears) != 1 || m.Hears[0].Consumer != "from-grants-2" {
+			return fmt.Errorf("the membership is %+v, want the one written once the state is there", m)
+		}
+		return nil
+	})
 }
 
 // novox/hq ADR 0363 §6 (issue 725): a consumer the membership names before the controller made it is bound once

node-tools/internal/meshtest/meshtest.go

@@ -131,6 +131,14 @@ func (m *Mesh) putGrants(t *testing.T, mem bus.Membership, body []byte) {
 	}
 }
 
+// DropGrants deletes the grants part's state memberships, as a bus that has not made it yet holds none; Issue makes it again.
+func (m *Mesh) DropGrants(t *testing.T) {
+	t.Helper()
+	if err := m.js.DeleteKeyValue(buscompose.GrantsMembershipsBucket); err != nil {
+		t.Fatal(err)
+	}
+}
+
 // sdkMembership is a membership of this package's type as mesh-sdk's, which the grants part composes and writes.
 func sdkMembership(t *testing.T, mem bus.Membership) membership.Membership {
 	t.Helper()