Bind a stream consumer again when the bus ends its loop, so a module is not left deaf when the controller deletes and makes again a consumer it cannot narrow (hq issue 725)

node-tools/internal/bus/streams.go

@@ -2,6 +2,7 @@ package bus
 
 import (
 	"context"
+	"errors"
 	"sort"
 	"strings"
 	"sync"
@@ -89,25 +90,35 @@ type hearing struct {
 	stopped bool
 	bound   map[HeardIssued]jetstream.ConsumeContext
 	said    map[HeardIssued]bool
-	kick    chan struct{}
-	done    chan struct{}
+	// why is the last error the bus gave a consumer's loop, said when the loop ends on it.
+	// Guarded by whyMu, not mu: the bus hands errors from its own goroutines, possibly while mu is held.
+	whyMu  sync.Mutex
+	why    map[HeardIssued]error
+	kick   chan struct{}
+	done   chan struct{}
+	unhear func()
 }
 
 func newHearing(c *Conn, js jetstream.JetStream, module string, deliver func(Envelope) error) *hearing {
 	return &hearing{c: c, js: js, module: module, deliver: deliver,
-		bound: map[HeardIssued]jetstream.ConsumeContext{}, said: map[HeardIssued]bool{},
+		bound: map[HeardIssued]jetstream.ConsumeContext{}, said: map[HeardIssued]bool{}, why: map[HeardIssued]error{},
 		kick: make(chan struct{}, 1), done: make(chan struct{})}
 }
 
-// bind reads one durable consumer by name, and on failure says what it was doing.
-func (h *hearing) bind(stream, durable string) (jetstream.ConsumeContext, string, error) {
+// bind reads one durable consumer by name, and on failure says what it was doing. Errors the bus gives
+// the loop afterwards are handed to onErr, when given.
+func (h *hearing) bind(stream, durable string, onErr func(error)) (jetstream.ConsumeContext, string, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 	consumer, err := h.js.Consumer(ctx, stream, durable)
 	if err != nil {
 		return nil, "binding", err
 	}
-	reading, err := consumer.Consume(h.handle, jetstream.PullMaxMessages(1))
+	opts := []jetstream.PullConsumeOpt{jetstream.PullMaxMessages(1)}
+	if onErr != nil {
+		opts = append(opts, jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { onErr(err) }))
+	}
+	reading, err := consumer.Consume(h.handle, opts...)
 	if err != nil {
 		return nil, "reading", err
 	}
@@ -159,14 +170,11 @@ func (h *hearing) handle(msg jetstream.Msg) {
 // what it no longer names, and a consumer not bound yet is tried again every RebindEvery.
 func (h *hearing) start() {
 	h.reconcile()
-	h.c.OnMembership(func(m Membership) {
+	h.unhear = h.c.onMembership(func(m Membership) {
 		if m.Module != h.module {
 			return
 		}
-		select {
-		case h.kick <- struct{}{}:
-		default:
-		}
+		h.nudge()
 	})
 	go func() {
 		tick := time.NewTicker(RebindEvery)
@@ -183,6 +191,21 @@ func (h *hearing) start() {
 	}()
 }
 
+// forget drops the last error a consumer's loop was given.
+func (h *hearing) forget(heard HeardIssued) {
+	h.whyMu.Lock()
+	delete(h.why, heard)
+	h.whyMu.Unlock()
+}
+
+// nudge asks the loop to reconcile now.
+func (h *hearing) nudge() {
+	select {
+	case h.kick <- struct{}{}:
+	default:
+	}
+}
+
 // stop stops every consumer on a module stream this module reads; the one on EVENTS is its caller's.
 func (h *hearing) stop() {
 	h.mu.Lock()
@@ -192,6 +215,9 @@ func (h *hearing) stop() {
 	}
 	h.stopped = true
 	close(h.done)
+	if h.unhear != nil {
+		h.unhear()
+	}
 	for heard, reading := range h.bound {
 		reading.Stop()
 		delete(h.bound, heard)
@@ -224,6 +250,26 @@ func (h *hearing) reconcile() {
 	for heard := range h.said {
 		if !want[heard] {
 			delete(h.said, heard)
+			h.forget(heard)
+		}
+	}
+	// **A loop the bus ended is no longer reading** (nats.go ends a pull loop on its own when its consumer is
+	// deleted): the controller deletes and makes again under the same name a restricted consumer it cannot
+	// narrow (ADR 0363 §6), and the membership, unchanged, says nothing — so it is bound again here.
+	for heard, reading := range h.bound {
+		select {
+		case <-reading.Closed():
+			delete(h.bound, heard)
+			h.whyMu.Lock()
+			why := h.why[heard]
+			h.whyMu.Unlock()
+			if why == nil {
+				why = errors.New("no reason given")
+			}
+			h.c.Logf("[mesh-tools] %s stopped hearing %s: the bus ended reading its consumer %s (%v); bound again as it exists",
+				h.module, heard.Stream, heard.Consumer, why)
+			h.said[heard] = true
+		default:
 		}
 	}
 	missing := make([]HeardIssued, 0, len(want))
@@ -234,7 +280,13 @@ func (h *hearing) reconcile() {
 	}
 	sort.Slice(missing, func(i, j int) bool { return missing[i].Stream < missing[j].Stream })
 	for _, heard := range missing {
-		reading, doing, err := h.bind(heard.Stream, heard.Consumer)
+		heard := heard
+		h.forget(heard)
+		reading, doing, err := h.bind(heard.Stream, heard.Consumer, func(err error) {
+			h.whyMu.Lock()
+			h.why[heard] = err
+			h.whyMu.Unlock()
+		})
 		if err != nil {
 			// Said once, not at every attempt; said again when it is bound.
 			if !h.said[heard] {
@@ -247,5 +299,13 @@ func (h *hearing) reconcile() {
 		h.bound[heard] = reading
 		delete(h.said, heard)
 		h.c.Logf("[mesh-tools] %s hears %s on its consumer %s", h.module, heard.Stream, heard.Consumer)
+		// When the bus ends the loop, look again at once rather than at the next tick.
+		go func() {
+			select {
+			case <-reading.Closed():
+				h.nudge()
+			case <-h.done:
+			}
+		}()
 	}
 }