The tool runner reads the memberships state again until it can, instead of giving a module up at its one follow

A JetStream 503 or a timeout at the follow (a bus right after its boot) left the module with no membership for good, since
Follow runs once per module and the stream fallback is gone. The read is now tried again with a growing wait (1 s doubling
to 30 s) until it watches the key or the connection closes; the failure is said once, and its reading once it succeeds.
Tests: none added; a step added to an existing unit test (TestAModuleHearsEveryStreamItsMembershipNames), which fails
without the retry.

(issue 774, 774/S32, review g1 round 1)
Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
This commit is contained in:
2026-11-02 00:51:19 +01:00
parent 5b2ff5e321
commit 331fa06b54
3 changed files with 80 additions and 12 deletions
+49 -12
View File
@@ -366,6 +366,13 @@ func (c *Conn) Follow(module string) {
// GrantsPatience bounds the first read of a membership from the grants part's state.
var GrantsPatience = 2 * time.Second
// GrantsRetry is the first wait before the memberships state is read again after it could not be; it doubles up to
// GrantsRetryMost.
var (
GrantsRetry = time.Second
GrantsRetryMost = 30 * time.Second
)
// issue applies a membership that arrived for module, live, and re-serves on it.
func (c *Conn) issue(module string, m Membership) {
c.mu.Lock()
@@ -383,9 +390,38 @@ func (c *Conn) issue(module string, m Membership) {
}
// followGrants reads module's key of the grants part's state memberships and watches it: the value there now is applied
// before Follow returns, each change after it live. A state that is not there yet, or not readable, is said once, and
// the module has no membership until it can be read.
// before Follow returns, each change after it live. A state that is not there yet, or not readable (a bus answering 503
// right after its boot, a timeout), is said once and read again, with a growing wait, until it can be or the
// connection closes: Follow runs once per module, so nothing else would ever read it again.
func (c *Conn) followGrants(module string) {
if c.watchGrants(module, true) {
return
}
go func() {
wait := GrantsRetry
for !c.nc.IsClosed() {
time.Sleep(wait)
if c.nc.IsClosed() {
return
}
if c.watchGrants(module, false) {
c.Logf("[mesh-tools] the grants part's memberships are read again; %s on %s follows its key", module, c.node)
return
}
if wait *= 2; wait > GrantsRetryMost {
wait = GrantsRetryMost
}
}
}()
}
// watchGrants is one attempt of followGrants: true once the key is watched. A failure is said when say is true.
func (c *Conn) watchGrants(module string, say bool) bool {
logf := func(format string, a ...any) {
if say {
c.Logf(format, a...)
}
}
key := buscompose.MembershipKey(c.node, module)
// First the key's direct get, which the account is granted alone: a bus that refuses it, or a state not there yet,
// answers at once, where binding the bucket would wait out its bound on a refusal.
@@ -396,27 +432,27 @@ func (c *Conn) followGrants(module string) {
if err != nil {
why = err.Error()
}
c.Logf("[mesh-tools] the grants part's memberships (%s) cannot be read (%s); %s has no membership "+
logf("[mesh-tools] the grants part's memberships (%s) cannot be read (%s); %s has no membership "+
"until it can be", bucket, why, module)
return
return false
}
js, err := jetstream.New(c.nc)
if err != nil {
return
return false
}
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
defer cancel()
kv, err := js.KeyValue(ctx, bucket)
if err != nil {
c.Logf("[mesh-tools] the grants part's memberships (%s) cannot be read (%v); %s has no membership "+
logf("[mesh-tools] the grants part's memberships (%s) cannot be read (%v); %s has no membership "+
"until it can be", buscompose.GrantsMembershipsBucket, err, module)
return
return false
}
w, err := kv.Watch(context.Background(), key)
if err != nil {
c.Logf("[mesh-tools] %s's key %s of the grants part's memberships cannot be watched (%v); %s has no membership "+
logf("[mesh-tools] %s's key %s of the grants part's memberships cannot be watched (%v); %s has no membership "+
"until it can be", module, key, err, module)
return
return false
}
apply := func(e jetstream.KeyValueEntry) {
if e.Operation() != jetstream.KeyValuePut {
@@ -443,7 +479,7 @@ func (c *Conn) followGrants(module string) {
select {
case e, ok := <-w.Updates():
if !ok {
return
return false
}
if e == nil {
current = false
@@ -452,9 +488,9 @@ func (c *Conn) followGrants(module string) {
apply(e)
case <-timer.C:
_ = w.Stop()
c.Logf("[mesh-tools] %s's key %s of the grants part's memberships did not answer in %s; %s has no membership "+
logf("[mesh-tools] %s's key %s of the grants part's memberships did not answer in %s; %s has no membership "+
"until it can be", module, key, StateTimeout, module)
return
return false
}
}
go func() {
@@ -464,6 +500,7 @@ func (c *Conn) followGrants(module string) {
}
}
}()
return true
}
// Membership is what the mesh issued a module here, or nil when nothing has been issued.