Refuse an event heard during a reload instead of panicking, and judge a bundle by its entrypoint alone (issue 1945, review)

An event a module's consumer heard after the module was dropped for a
reload read a deleted entry and panicked node-tools, taking every tool on
the machine down. The handler now refuses an event of a consumer that is no
longer the module's, and a stopped hearing hands nothing on and offers the
event again at once.

A bundle that writes beside itself (a cache, a socket) loaded itself again
after every quiet spell, because the signature took in every file in its
directory. A delivery puts every file in place anew, so the entrypoint alone
tells a new build. The watch also looks once as it begins, so a bundle that
came between its load and the watch is not missed.

Tests: 1945/AC23 and AC24 in internal/runtime/reload_test.go, and a stopped
hearing in internal/bus/hearing_stopped_test.go.

node-tools/internal/bus/hearing_stopped_test.go

@@ -0,0 +1,48 @@
+package bus
+
+import (
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// heardMsg is a delivered event as a consumer's loop hands it, saying how it was settled.
+type heardMsg struct {
+	jetstream.Msg
+	mu      sync.Mutex
+	settled []string
+}
+
+func (m *heardMsg) Subject() string                  { return "mesh.mod.alpha.event.happened" }
+func (m *heardMsg) Headers() nats.Header             { return nats.Header{} }
+func (m *heardMsg) Data() []byte                     { return []byte(`{"n":1}`) }
+func (m *heardMsg) InProgress() error                { return nil }
+func (m *heardMsg) Ack() error                       { return m.say("ack") }
+func (m *heardMsg) Nak() error                       { return m.say("nak") }
+func (m *heardMsg) NakWithDelay(time.Duration) error { return m.say("nak-delayed") }
+func (m *heardMsg) Term() error                      { return m.say("term") }
+func (m *heardMsg) say(how string) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.settled = append(m.settled, how)
+	return nil
+}
+
+// An event the bus hands a hearing after it was stopped (a module's bundle loaded again, issue 1945) is
+// offered again at once and never handed to the module: the module's handler that heard it may be gone.
+func TestAHearingStoppedHandsNoEventOnAndOffersItAgain(t *testing.T) {
+	delivered := 0
+	h := newHearing(nil, nil, "alpha", func(Envelope) error { delivered++; return nil })
+	h.stop()
+	msg := &heardMsg{}
+	h.handle(msg)
+	if delivered != 0 {
+		t.Errorf("a stopped hearing handed the event to the module")
+	}
+	if len(msg.settled) != 1 || msg.settled[0] != "nak" {
+		t.Errorf("a stopped hearing settled the event as %v, want it offered again at once (nak)", msg.settled)
+	}
+}

node-tools/internal/bus/streams.go

@@ -6,6 +6,7 @@ import (
 	"sort"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/nats-io/nats.go/jetstream"
@@ -86,6 +87,9 @@ type hearing struct {
 	// serial hands the module one event at a time across its consumers (novox/hq issue 387).
 	serial sync.Mutex
 
+	// halted is stopped, read by handle without mu: the bus's loops hand events while mu is held.
+	halted atomic.Bool
+
 	mu      sync.Mutex
 	stopped bool
 	bound   map[HeardIssued]jetstream.ConsumeContext
@@ -154,11 +158,21 @@ func (h *hearing) handle(msg jetstream.Msg) {
 		}
 	}()
 	h.serial.Lock()
-	err := h.deliver(env)
+	halted := h.halted.Load()
+	var err error
+	if !halted {
+		err = h.deliver(env)
+	}
 	h.serial.Unlock()
 	close(done)
 	// No in-progress after the answer: one after a Nak would stretch its delay to the whole ack wait.
 	<-stopped
+	if halted {
+		// **A hearing stopped hands nothing on** (issue 1945): the module's bundle that subscribed may be
+		// gone, loaded again in its place; offered again at once, to the consumer bound after it.
+		_ = msg.Nak()
+		return
+	}
 	if err != nil {
 		_ = msg.NakWithDelay(NakDelay)
 		return
@@ -214,6 +228,7 @@ func (h *hearing) stop() {
 		return
 	}
 	h.stopped = true
+	h.halted.Store(true)
 	close(h.done)
 	if h.unhear != nil {
 		h.unhear()