Keep what a consumer gives up on until a person delivers it again or drops it

Design 25 promised a dead-letter stream that did not exist: a message a
consumer gave up on stayed only in its source, which drops it after a week,
and its condition cleared when the advisories stopped (hq issue 330, ADR 0264).

cmd/mesh-controller/signals.go

@@ -154,8 +154,9 @@ var signalsTable = []signalRow{
 		}},
 	{Row: "S9", Signal: "bus advisories: maximum deliveries, consumer deleted; the controller's own slow " +
 		"consumer and refused subjects", Emitter: "bus server's advisory subjects; the controller's connection",
-		Trigger: "any", Bound: "any occurrence; clears after an hour without another, and a deleted consumer " +
-			"once it exists again or the mesh no longer expects it",
+		Trigger: "any", Bound: "any occurrence; clears after an hour without another, a deleted consumer " +
+			"once it exists again or the mesh no longer expects it, and a message given up on once DEAD_LETTERS " +
+			"no longer holds it (novox/hq issue 330)",
 		Kind: "slow-consumer, max-deliveries, refused, consumer-lost", Severity: conditions.Warning, Phase: 1,
 		needs: func(f *signalFacts) error { return f.advisoriesErr }, watch: watchAdvisories,
 		newest: func(f *signalFacts) time.Time {
@@ -485,12 +486,94 @@ func watchAdvisories(f *signalFacts) []conditions.Observation {
 		if a.ID == "controller" {
 			machine = f.host
 		}
-		out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: a.ID, Kind: a.Kind,
-			Machine: machine, Severity: severity, Summary: a.Said + times, Said: a.Said})
+		o := conditions.Observation{Scope: conditions.ScopeBus, ID: a.ID, Kind: a.Kind, Token: a.Token,
+			Machine: machine, Severity: severity, Summary: a.Said + times, Said: a.Said}
+		if a.Kind == link.AdvisoryMaxDeliveries && a.Token == link.AdvisoryNotKept {
+			o.Machine = consumerMachine(a.Stream, a.Consumer)
+			o.Headline = clip(conditions.Capital(fmt.Sprintf("%s gave up on a message, not kept",
+				consumerWho(a.Stream, a.Consumer))), 60)
+			o.Explanation = "A listener on the bus could not handle a message, and the mesh could not keep it " +
+				"for you yet. It tries again every minute."
+			o.Resolved = "Resolved: the message is kept"
+		}
+		out = append(out, o)
+	}
+	return append(out, watchDeadLetters(f)...)
+}
+
+// watchDeadLetters says each consumer that DEAD_LETTERS holds a message for (novox/hq issue 330): open
+// while it holds any, so it clears when they are delivered again or dropped, never because the server
+// stopped saying it.
+func watchDeadLetters(f *signalFacts) []conditions.Observation {
+	keys := make([]string, 0, len(f.deadLetters))
+	for k := range f.deadLetters {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	var out []conditions.Observation
+	for _, key := range keys {
+		n := f.deadLetters[key]
+		stream, consumer, _ := strings.Cut(key, ".")
+		messages, them := "a message", "it"
+		if n > 1 {
+			messages, them = fmt.Sprintf("%d messages", n), "them"
+		}
+		who := consumerWho(stream, consumer)
+		out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: key, Kind: link.AdvisoryMaxDeliveries,
+			Machine: consumerMachine(stream, consumer), Severity: conditions.Warning,
+			Summary: fmt.Sprintf("%s gave up on %s; %s kept in %s until delivered again or dropped, with why, "+
+				"through the controller's dead-letters verb", link.ConsumerInWords(stream, consumer), messages,
+				map[bool]string{true: "they are", false: "it is"}[n > 1], broker.DeadLettersStream),
+			Said:     fmt.Sprintf("%d held for %s", n, key),
+			Headline: clip(conditions.Capital(fmt.Sprintf("%s could not handle %s", who, messages)), 60),
+			Needs:    fmt.Sprintf("deliver %s again or drop %s, from the mesh MCP server.", them, them),
+			Explanation: conditions.Capital(fmt.Sprintf("%s was handed %s several times and gave up, so what %s "+
+				"asked for was not done. The mesh keeps %s until you deliver %s again or drop %s.", who, messages,
+				them, them, them, them)),
+			Resolved: "Resolved: the messages it gave up on were delivered again or dropped"})
 	}
 	return out
 }
 
+// consumerWho is a durable consumer's holder as the operator says it: a module on its machine, the
+// controller, or a seat's holders.
+func consumerWho(stream, consumer string) string {
+	switch {
+	case consumer == broker.ControllerName:
+		return "the controller"
+	case strings.HasPrefix(stream, "SEAT_") && strings.HasSuffix(consumer, "_worker"):
+		seat := strings.ToLower(strings.ReplaceAll(strings.TrimSuffix(strings.TrimPrefix(consumer, "SEAT_"), "_worker"), "_", "-"))
+		return "the holder of " + seat
+	case stream == broker.EventsStream:
+		if node, module, ok := strings.Cut(consumer, "_"); ok {
+			return module + " on " + node
+		}
+	}
+	return "a listener on the bus"
+}
+
+// consumerMachine is the machine a module's consumer is on; empty for the others.
+func consumerMachine(stream, consumer string) string {
+	if stream == broker.EventsStream {
+		if node, _, ok := strings.Cut(consumer, "_"); ok {
+			return node
+		}
+	}
+	return ""
+}
+
+// clip is words at most n characters long, cut at a word.
+func clip(s string, n int) string {
+	if len(s) <= n {
+		return s
+	}
+	cut := s[:n]
+	if i := strings.LastIndex(cut, " "); i > 0 {
+		cut = cut[:i]
+	}
+	return cut
+}
+
 func watchSelfCheck(f *signalFacts) []conditions.Observation {
 	every := f.selfCheck.every
 	if every <= 0 {