Merge pull request 'Keep what a consumer gives up on until a person delivers it again or drops it (hq issue 330, ADR 0264)' (#159) from fix/330-a-message-given-up-on-is-kept into main

cmd/mesh-controller/deadletters.go

@@ -0,0 +1,166 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+	"sync/atomic"
+
+	"github.com/nats-io/nats.go"
+
+	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// What a consumer gave up on, answered by the serving controller (novox/hq issue 330).
+//
+// **In this process, on its own connection**: DEAD_LETTERS is read and changed on the bus, and the
+// serving controller is already on it. A verb run as a fresh process would open a connection of its own
+// for each call (novox/hq issue 327).
+
+// deadLettersOn is the serving controller's bus, set when it starts watching the mesh; nil in a
+// process that serves nothing.
+var deadLettersOn atomic.Pointer[busHandles]
+
+// busHandles are the serving controller's connection and JetStream handle.
+type busHandles struct {
+	conn *nats.Conn
+	js   nats.JetStreamContext
+}
+
+// defaultDeadLetters is how many the list says when not asked for more.
+const defaultDeadLetters = 50
+
+// causeDeadLetter is the cause a delivery or drop gives when the caller gives none.
+const causeDeadLetter = "dead-letter"
+
+// deadLettersAnswer is what `dead-letters` answers: the list, one whole, or what came of delivering one
+// again or dropping it.
+func deadLettersAnswer(ctx context.Context, a *verbArguments) (any, error) {
+	on := deadLettersOn.Load()
+	if on == nil {
+		return nil, errors.New("this controller is not serving, so it does not read DEAD_LETTERS: ask again, " +
+			"and the serving controller answers")
+	}
+	// The shape first, and every argument it reads; one given beside it is refused before anything is
+	// done, as every verb refuses what it would pass over (novox/hq issue 244).
+	var deliver, drop, why, cause, idText, consumer, limit string
+	switch {
+	case a.given["deliver"] != "" || a.given["drop"] != "":
+		deliver, drop, why, cause = a.str("deliver"), a.str("drop"), a.str("why"), a.str("cause")
+	case a.given["id"] != "":
+		idText = a.str("id")
+	default:
+		consumer, limit = a.str("consumer"), a.str("limit")
+	}
+	if unused := a.unused(); len(unused) > 0 {
+		return nil, fmt.Errorf("dead-letters did not use %s together with %s, and an argument a verb would pass "+
+			"over is refused: nothing was done", quoteAll(unused), quoteAll(a.usedGiven()))
+	}
+	switch {
+	case deliver != "" && drop != "":
+		return nil, errors.New("dead-letters delivers one again or drops one, not both. Nothing was done")
+	case deliver != "" || drop != "":
+		act, text := "deliver", deliver
+		if drop != "" {
+			act, text = "drop", drop
+		}
+		if strings.TrimSpace(why) == "" {
+			return nil, fmt.Errorf("dead-letters %s is a hand act, and says why: why is required and recorded in "+
+				"the hand-act log (novox/hq to-be 45 §7). Nothing was done", act)
+		}
+		id, err := deadLetterID(text)
+		if err != nil {
+			return nil, err
+		}
+		return actOnDeadLetter(ctx, on, act, id, why, cause)
+	case idText != "":
+		id, err := deadLetterID(idText)
+		if err != nil {
+			return nil, err
+		}
+		return link.DeadLetterNamed(on.js, id)
+	}
+	most := defaultDeadLetters
+	if limit != "" {
+		n, err := strconv.Atoi(limit)
+		if err != nil || n <= 0 {
+			return nil, fmt.Errorf("limit is a number of dead letters, not %q", limit)
+		}
+		most = n
+	}
+	held, total, err := link.DeadLetters(on.js, consumer, most)
+	if err != nil {
+		return nil, err
+	}
+	answer := map[string]any{"dead_letters": held, "held": total,
+		"note": "newest first; with id, one whole; deliver or drop one with why"}
+	if total == 0 {
+		answer["note"] = "no consumer gave up on a message that is still kept"
+	}
+	return answer, nil
+}
+
+// deadLetterID is a dead letter's id as a caller wrote it.
+func deadLetterID(text string) (uint64, error) {
+	id, err := strconv.ParseUint(strings.TrimSpace(text), 10, 64)
+	if err != nil || id == 0 {
+		return 0, fmt.Errorf("a dead letter's id is its number in %s, as dead-letters lists it, not %q",
+			broker.DeadLettersStream, text)
+	}
+	return id, nil
+}
+
+// actOnDeadLetter delivers one again or drops it, recorded in the hand-act log before it is done. A log
+// that cannot be written is said, and the act still happens: the log is never the reason a person's act
+// is refused (handacts.go).
+func actOnDeadLetter(ctx context.Context, on *busHandles, act string, id uint64, why, cause string) (any, error) {
+	d, err := link.DeadLetterNamed(on.js, id)
+	if err != nil {
+		return nil, err
+	}
+	if act == "deliver" {
+		// Refused before it is recorded: an act that cannot be done is not an act.
+		if _, err := link.AgainTo(on.js, d); err != nil {
+			return nil, err
+		}
+	}
+	if cause == "" {
+		cause = causeDeadLetter
+	}
+	by := link.CallerIn(ctx)
+	if by == "" {
+		by = "a seat call whose caller the bus did not name"
+	}
+	answer := map[string]any{"dead_letter": d.ID, "consumer": d.Who, "subject": d.Subject}
+	recorded, logErr := link.RecordHandAct(ctx, on.conn, link.HandAct{Verb: "dead-letters " + act,
+		Args: []string{strconv.FormatUint(id, 10), d.Stream + "." + d.Consumer}, Why: why, Cause: cause,
+		Condition: conditions.Key(conditions.ScopeBus, d.Stream+"."+d.Consumer, link.AdvisoryMaxDeliveries),
+		By:        by + ", through the " + catalogue.ControllerSeatName + " seat"})
+	if logErr != nil {
+		answer["unrecorded"] = "the hand-act log could not be written, and the act was done all the same: " + logErr.Error()
+	} else {
+		answer["recorded"] = recorded.ID
+	}
+	switch act {
+	case "deliver":
+		_, to, err := link.DeliverAgain(on.js, id)
+		if err != nil {
+			return nil, err
+		}
+		answer["delivered_on"] = to
+		answer["done"] = fmt.Sprintf("dead letter %d was delivered again to %s, and nobody else; it is no longer kept",
+			id, consumerWho(d.Stream, d.Consumer))
+	case "drop":
+		if _, err := link.DropDeadLetter(on.js, id); err != nil {
+			return nil, err
+		}
+		answer["done"] = fmt.Sprintf("dead letter %d, which %s gave up on, was dropped for good", id,
+			consumerWho(d.Stream, d.Consumer))
+	}
+	return answer, nil
+}

cmd/mesh-controller/deadletters_test.go

@@ -0,0 +1,132 @@
+package main
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/link"
+	"github.com/novox/mesh-controller/internal/testbus"
+)
+
+// The serving controller's bus, with the mesh's streams, for the verb to read and act on.
+func servingDeadLetters(t *testing.T) *broker.JetStream {
+	t.Helper()
+	js, err := broker.Dial(testbus.URL(t))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(js.Close)
+	if err := broker.AssertMeshStreams(js); err != nil {
+		t.Fatal(err)
+	}
+	if err := js.EnsureControllerBuckets(); err != nil {
+		t.Fatal(err)
+	}
+	before := deadLettersOn.Load()
+	deadLettersOn.Store(&busHandles{conn: js.Conn(), js: js.Context()})
+	t.Cleanup(func() { deadLettersOn.Store(before) })
+	return js
+}
+
+func askDeadLetters(t *testing.T, args map[string]any) (any, error) {
+	t.Helper()
+	a, err := readArguments("dead-letters", args)
+	if err != nil {
+		return nil, err
+	}
+	return deadLettersAnswer(context.Background(), a)
+}
+
+func TestDeadLettersListsDropsAndRecordsWhy(t *testing.T) {
+	js := servingDeadLetters(t)
+	if _, err := js.Context().Publish("mesh.mod.gitea.event.pull.merged", []byte(`{"n":1}`)); err != nil {
+		t.Fatal(err)
+	}
+	d, err := link.KeepDeadLetter(js.Context(), []byte(`{"stream":"EVENTS","consumer":"media_sonarr","stream_seq":1,"deliveries":5}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	answer, err := askDeadLetters(t, map[string]any{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	listed := answer.(map[string]any)
+	if listed["held"] != 1 || len(listed["dead_letters"].([]link.DeadLetter)) != 1 {
+		t.Fatalf("listed %v", listed)
+	}
+
+	// Refused before anything is done: an act without why, an argument the shape passes over, both acts.
+	for _, args := range []map[string]any{
+		{"drop": "1"},
+		{"drop": "1", "why": "x", "limit": "3"},
+		{"drop": "1", "deliver": "1", "why": "x"},
+		{"why": "x"},
+		{"id": "nought"},
+	} {
+		if _, err := askDeadLetters(t, args); err == nil {
+			t.Errorf("%v was done", args)
+		}
+	}
+	if _, total, _ := link.DeadLetters(js.Context(), "", 0); total != 1 {
+		t.Fatalf("a refused call changed what is kept: %d left", total)
+	}
+
+	done, err := askDeadLetters(t, map[string]any{"drop": "1", "why": "the media server took the download in by hand"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if said := done.(map[string]any); said["recorded"] == nil || !strings.Contains(said["done"].(string), "dropped") {
+		t.Fatalf("answered %v", said)
+	}
+	acts, err := link.HandActs(context.Background(), js.Conn(), time.Now().Add(-time.Minute))
+	if err != nil || len(acts) != 1 || acts[0].Verb != "dead-letters drop" || acts[0].Cause != causeDeadLetter ||
+		!strings.Contains(acts[0].Condition, "media_sonarr") {
+		t.Fatalf("recorded %+v (%v)", acts, err)
+	}
+	if !personsDecision(acts[0]) {
+		t.Error("dropping a dead letter is counted as a repair, so S15 would want a healer for it")
+	}
+	if _, err := askDeadLetters(t, map[string]any{"id": "1"}); err == nil {
+		t.Errorf("dead letter %d is still answered after it was dropped", d.ID)
+	}
+}
+
+// Open while DEAD_LETTERS holds a message for the consumer, in words the operator reads in one pass:
+// what is held, and where to act.
+func TestAConsumersDeadLettersAreSaidUntilActedOn(t *testing.T) {
+	f := &signalFacts{now: time.Now(), deadLetters: map[string]int{"EVENTS.media_sonarr": 4, "EVENTS.controller": 1}}
+	found := watchDeadLetters(f)
+	if len(found) != 2 {
+		t.Fatalf("said %d conditions", len(found))
+	}
+	for _, o := range found {
+		if o.Kind != "max-deliveries" || o.Severity != conditions.Warning || o.Needs == "" {
+			t.Errorf("%+v", o)
+		}
+		if why, ok := conditions.PlainWords(conditions.Words{Headline: o.Headline, Needs: o.Needs,
+			Explanation: o.Explanation, Resolved: o.Resolved}, "media"); !ok {
+			t.Errorf("%q is not plain: %s", o.Headline, why)
+		}
+		if !strings.Contains(o.Needs, "mesh MCP server") {
+			t.Errorf("does not say where to act: %q", o.Needs)
+		}
+	}
+	sonarr := found[1]
+	if sonarr.ID != "EVENTS.media_sonarr" || sonarr.Machine != "media" ||
+		sonarr.Headline != "Sonarr on media could not handle 4 messages" ||
+		!strings.Contains(sonarr.Summary, "DEAD_LETTERS") {
+		t.Errorf("%+v", sonarr)
+	}
+	if found[0].Headline != "The controller could not handle a message" {
+		t.Errorf("%q", found[0].Headline)
+	}
+	// None held, none said: it clears when they are delivered again or dropped.
+	if left := watchDeadLetters(&signalFacts{now: time.Now()}); len(left) != 0 {
+		t.Fatalf("%v", left)
+	}
+}

cmd/mesh-controller/handacts.go

@@ -82,6 +82,11 @@ var handActVerbs = []handActVerb{
 	{Verb: "retire approve", Decision: "nothing is retired past its bound without a person (ADR 0230)"},
 	{Verb: "retire reject", Decision: "keeping a consumer active is a person's word (ADR 0230)"},
 	{Verb: "cleanup delete", Decision: "nothing retired is deleted without a person (ADR 0230)"},
+	// What becomes of a message a consumer gave up on (novox/hq issue 330): kept until a person says.
+	{Verb: "dead-letters deliver", Decision: "a message a consumer gave up on is delivered again only on a " +
+		"person's word (issue 330)"},
+	{Verb: "dead-letters drop", Decision: "a message a consumer gave up on is let go only on a person's word " +
+		"(issue 330)"},
 	// The sweep run on a person's word rather than after a build: the same decision the records make, at
 	// a moment the person chose (ADR 0251) — never a repair.
 	{Verb: "collect", Decision: "letting the store go of what the records keep for no reason, now rather " +

cmd/mesh-controller/plain_words.go

@@ -544,9 +544,11 @@ var plainWordings = map[string]func(conditions.Observation) words{
 			Resolved:    "The listener keeps up again"}
 	}),
 	"max-deliveries": worded(func(o conditions.Observation) words {
-		return words{Headline: "A message could not be handled",
-			Explanation: "The bus gave up on a message after trying to hand it over too many times.",
-			Resolved:    "Resolved: messages are handled again"}
+		return words{Headline: "A listener gave up on messages",
+			Needs: "deliver them again or drop them, from the mesh MCP server.",
+			Explanation: "A listener on the bus could not handle messages after several tries, so what they asked " +
+				"for was not done. The mesh keeps them until you deliver them again or drop them.",
+			Resolved: "Resolved: the messages given up on were delivered again or dropped"}
 	}),
 	"refused": worded(func(o conditions.Observation) words {
 		return words{Headline: "The bus refuses some messages",

cmd/mesh-controller/probes.go

@@ -875,6 +875,16 @@ func streamDiffers(want broker.Stream, have nats.StreamConfig) string {
 	if perSubject != 0 && have.MaxMsgsPerSubject != perSubject {
 		differs = append(differs, fmt.Sprintf("keeps %d per subject, defined %d", have.MaxMsgsPerSubject, perSubject))
 	}
+	if want.MaxBytes > 0 && have.MaxBytes != want.MaxBytes {
+		differs = append(differs, fmt.Sprintf("holds up to %d bytes, defined %d", have.MaxBytes, want.MaxBytes))
+	}
+	if want.DuplicatesSeconds > 0 && have.Duplicates != time.Duration(want.DuplicatesSeconds)*time.Second {
+		differs = append(differs, fmt.Sprintf("keeps one of a message id for %s, defined %s", have.Duplicates,
+			time.Duration(want.DuplicatesSeconds)*time.Second))
+	}
+	if want.DiscardNew && have.Discard != nats.DiscardNew {
+		differs = append(differs, "drops what it holds when full, defined to refuse what comes next")
+	}
 	return strings.Join(differs, "; ")
 }
 

cmd/mesh-controller/seatverbs.go

@@ -250,6 +250,8 @@ func (a *verbArguments) commandLine() ([]string, error) {
 		return argv, nil
 	case "tools":
 		return nil, errors.New("tools is answered from the records, not by a command")
+	case "dead-letters":
+		return nil, errors.New("dead-letters is answered by the serving controller, on its own connection, not by a command")
 	case "status":
 		return []string{"status", "--json"}, nil
 	case "nodes":
@@ -982,6 +984,9 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
 				if verb == "calls" {
 					return callsAnswer(link.Calls, a.given["call"])
 				}
+				if verb == "dead-letters" {
+					return deadLettersAnswer(ctx, a)
+				}
 				if verb == "doctor" {
 					// From the serving controller, which runs the self-check and hears the signals
 					// (novox/hq to-be 45 §4): the last verdict at once, or a run now.
@@ -1064,7 +1069,10 @@ func actsOnAPlan(args map[string]any) bool {
 
 // inProcess are the verbs answered by this process rather than by a command it runs: `tools` from
 // the records, `calls` from what this process served.
-var inProcess = map[string]bool{"tools": true, "calls": true, "doctor": true}
+var inProcess = map[string]bool{"tools": true, "calls": true, "doctor": true,
+	// What a consumer gave up on, read and changed on the serving controller's own connection (novox/hq
+	// issue 330).
+	"dead-letters": true}
 
 // answersFirst is a command line whose caller is answered before it runs: a push, by its verb or
 // through `command`. A push sends the machine holding the bus first when its user list changed, the

cmd/mesh-controller/signals.go

@@ -154,8 +154,9 @@ var signalsTable = []signalRow{
 		}},
 	{Row: "S9", Signal: "bus advisories: maximum deliveries, consumer deleted; the controller's own slow " +
 		"consumer and refused subjects", Emitter: "bus server's advisory subjects; the controller's connection",
-		Trigger: "any", Bound: "any occurrence; clears after an hour without another, and a deleted consumer " +
-			"once it exists again or the mesh no longer expects it",
+		Trigger: "any", Bound: "any occurrence; clears after an hour without another, a deleted consumer " +
+			"once it exists again or the mesh no longer expects it, and a message given up on once DEAD_LETTERS " +
+			"no longer holds it (novox/hq issue 330)",
 		Kind: "slow-consumer, max-deliveries, refused, consumer-lost", Severity: conditions.Warning, Phase: 1,
 		needs: func(f *signalFacts) error { return f.advisoriesErr }, watch: watchAdvisories,
 		newest: func(f *signalFacts) time.Time {
@@ -485,12 +486,94 @@ func watchAdvisories(f *signalFacts) []conditions.Observation {
 		if a.ID == "controller" {
 			machine = f.host
 		}
-		out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: a.ID, Kind: a.Kind,
-			Machine: machine, Severity: severity, Summary: a.Said + times, Said: a.Said})
+		o := conditions.Observation{Scope: conditions.ScopeBus, ID: a.ID, Kind: a.Kind, Token: a.Token,
+			Machine: machine, Severity: severity, Summary: a.Said + times, Said: a.Said}
+		if a.Kind == link.AdvisoryMaxDeliveries && a.Token == link.AdvisoryNotKept {
+			o.Machine = consumerMachine(a.Stream, a.Consumer)
+			o.Headline = clip(conditions.Capital(fmt.Sprintf("%s gave up on a message, not kept",
+				consumerWho(a.Stream, a.Consumer))), 60)
+			o.Explanation = "A listener on the bus could not handle a message, and the mesh could not keep it " +
+				"for you yet. It tries again every minute."
+			o.Resolved = "Resolved: the message is kept"
+		}
+		out = append(out, o)
+	}
+	return append(out, watchDeadLetters(f)...)
+}
+
+// watchDeadLetters says each consumer that DEAD_LETTERS holds a message for (novox/hq issue 330): open
+// while it holds any, so it clears when they are delivered again or dropped, never because the server
+// stopped saying it.
+func watchDeadLetters(f *signalFacts) []conditions.Observation {
+	keys := make([]string, 0, len(f.deadLetters))
+	for k := range f.deadLetters {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	var out []conditions.Observation
+	for _, key := range keys {
+		n := f.deadLetters[key]
+		stream, consumer, _ := strings.Cut(key, ".")
+		messages, them := "a message", "it"
+		if n > 1 {
+			messages, them = fmt.Sprintf("%d messages", n), "them"
+		}
+		who := consumerWho(stream, consumer)
+		out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: key, Kind: link.AdvisoryMaxDeliveries,
+			Machine: consumerMachine(stream, consumer), Severity: conditions.Warning,
+			Summary: fmt.Sprintf("%s gave up on %s; %s kept in %s until delivered again or dropped, with why, "+
+				"through the controller's dead-letters verb", link.ConsumerInWords(stream, consumer), messages,
+				map[bool]string{true: "they are", false: "it is"}[n > 1], broker.DeadLettersStream),
+			Said:     fmt.Sprintf("%d held for %s", n, key),
+			Headline: clip(conditions.Capital(fmt.Sprintf("%s could not handle %s", who, messages)), 60),
+			Needs:    fmt.Sprintf("deliver %s again or drop %s, from the mesh MCP server.", them, them),
+			Explanation: conditions.Capital(fmt.Sprintf("%s was handed %s several times and gave up, so what %s "+
+				"asked for was not done. The mesh keeps %s until you deliver %s again or drop %s.", who, messages,
+				them, them, them, them)),
+			Resolved: "Resolved: the messages it gave up on were delivered again or dropped"})
 	}
 	return out
 }
 
+// consumerWho is a durable consumer's holder as the operator says it: a module on its machine, the
+// controller, or a seat's holders.
+func consumerWho(stream, consumer string) string {
+	switch {
+	case consumer == broker.ControllerName:
+		return "the controller"
+	case strings.HasPrefix(stream, "SEAT_") && strings.HasSuffix(consumer, "_worker"):
+		seat := strings.ToLower(strings.ReplaceAll(strings.TrimSuffix(strings.TrimPrefix(consumer, "SEAT_"), "_worker"), "_", "-"))
+		return "the holder of " + seat
+	case stream == broker.EventsStream:
+		if node, module, ok := strings.Cut(consumer, "_"); ok {
+			return module + " on " + node
+		}
+	}
+	return "a listener on the bus"
+}
+
+// consumerMachine is the machine a module's consumer is on; empty for the others.
+func consumerMachine(stream, consumer string) string {
+	if stream == broker.EventsStream {
+		if node, _, ok := strings.Cut(consumer, "_"); ok {
+			return node
+		}
+	}
+	return ""
+}
+
+// clip is words at most n characters long, cut at a word.
+func clip(s string, n int) string {
+	if len(s) <= n {
+		return s
+	}
+	cut := s[:n]
+	if i := strings.LastIndex(cut, " "); i > 0 {
+		cut = cut[:i]
+	}
+	return cut
+}
+
 func watchSelfCheck(f *signalFacts) []conditions.Observation {
 	every := f.selfCheck.every
 	if every <= 0 {

cmd/mesh-controller/watchdogs.go

@@ -75,6 +75,9 @@ type signalFacts struct {
 
 	advisories    []link.Advisory
 	lostConsumers map[string]bool
+	// deadLetters are how many messages DEAD_LETTERS holds per consumer, by `<stream>.<consumer>`
+	// (novox/hq issue 330): each consumer's max-deliveries condition is open while it holds any.
+	deadLetters   map[string]int
 	advisoriesErr error
 
 	selfCheck selfCheckFacts
@@ -332,6 +335,9 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts {
 	}
 	f.advisories = link.Advisories.Since(now.Add(-advisoryQuiet))
 	f.lostConsumers, f.advisoriesErr = w.lostConsumers(ctx, f.advisories)
+	if f.advisoriesErr == nil && w.js != nil {
+		f.deadLetters, f.advisoriesErr = link.HeldDeadLetters(w.js.Context())
+	}
 	f.handActs, f.handActsErr = w.gatherHandActs(ctx, now)
 	f.facts.taken, _, f.facts.began, f.facts.err = exportedFacts.last()
 	return f
@@ -654,6 +660,8 @@ func watchTheMesh(ctx context.Context, open *stores, server *link.Server, bus li
 		return nil
 	}
 	conditionsFrom = keeper
+	// What a consumer gave up on is read and acted on by the verb in this process (novox/hq issue 330).
+	deadLettersOn.Store(&busHandles{conn: server.JetStream().Conn(), js: server.JetStream().Context()})
 	logf := func(format string, args ...any) { fmt.Printf(format+"\n", args...) }
 	stopHearing, err := server.HearAdvisories(logf)
 	if err != nil {
@@ -680,6 +688,8 @@ func watchTheMesh(ctx context.Context, open *stores, server *link.Server, bus li
 	return func() {
 		stop()
 		stopHearing()
+		// No longer serving: the verb answers that it does not read DEAD_LETTERS here (novox/hq issue 330).
+		deadLettersOn.Store(nil)
 		flushing, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 		defer cancel()
 		keeper.Close(flushing)