Keep what a consumer gives up on until a person delivers it again or drops it

Design 25 promised a dead-letter stream that did not exist: a message a
consumer gave up on stayed only in its source, which drops it after a week,
and its condition cleared when the advisories stopped (hq issue 330, ADR 0264).

cmd/mesh-controller/deadletters_test.go

@@ -0,0 +1,132 @@
+package main
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/link"
+	"github.com/novox/mesh-controller/internal/testbus"
+)
+
+// The serving controller's bus, with the mesh's streams, for the verb to read and act on.
+func servingDeadLetters(t *testing.T) *broker.JetStream {
+	t.Helper()
+	js, err := broker.Dial(testbus.URL(t))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(js.Close)
+	if err := broker.AssertMeshStreams(js); err != nil {
+		t.Fatal(err)
+	}
+	if err := js.EnsureControllerBuckets(); err != nil {
+		t.Fatal(err)
+	}
+	before := deadLettersOn.Load()
+	deadLettersOn.Store(&busHandles{conn: js.Conn(), js: js.Context()})
+	t.Cleanup(func() { deadLettersOn.Store(before) })
+	return js
+}
+
+func askDeadLetters(t *testing.T, args map[string]any) (any, error) {
+	t.Helper()
+	a, err := readArguments("dead-letters", args)
+	if err != nil {
+		return nil, err
+	}
+	return deadLettersAnswer(context.Background(), a)
+}
+
+func TestDeadLettersListsDropsAndRecordsWhy(t *testing.T) {
+	js := servingDeadLetters(t)
+	if _, err := js.Context().Publish("mesh.mod.gitea.event.pull.merged", []byte(`{"n":1}`)); err != nil {
+		t.Fatal(err)
+	}
+	d, err := link.KeepDeadLetter(js.Context(), []byte(`{"stream":"EVENTS","consumer":"media_sonarr","stream_seq":1,"deliveries":5}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	answer, err := askDeadLetters(t, map[string]any{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	listed := answer.(map[string]any)
+	if listed["held"] != 1 || len(listed["dead_letters"].([]link.DeadLetter)) != 1 {
+		t.Fatalf("listed %v", listed)
+	}
+
+	// Refused before anything is done: an act without why, an argument the shape passes over, both acts.
+	for _, args := range []map[string]any{
+		{"drop": "1"},
+		{"drop": "1", "why": "x", "limit": "3"},
+		{"drop": "1", "deliver": "1", "why": "x"},
+		{"why": "x"},
+		{"id": "nought"},
+	} {
+		if _, err := askDeadLetters(t, args); err == nil {
+			t.Errorf("%v was done", args)
+		}
+	}
+	if _, total, _ := link.DeadLetters(js.Context(), "", 0); total != 1 {
+		t.Fatalf("a refused call changed what is kept: %d left", total)
+	}
+
+	done, err := askDeadLetters(t, map[string]any{"drop": "1", "why": "the media server took the download in by hand"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if said := done.(map[string]any); said["recorded"] == nil || !strings.Contains(said["done"].(string), "dropped") {
+		t.Fatalf("answered %v", said)
+	}
+	acts, err := link.HandActs(context.Background(), js.Conn(), time.Now().Add(-time.Minute))
+	if err != nil || len(acts) != 1 || acts[0].Verb != "dead-letters drop" || acts[0].Cause != causeDeadLetter ||
+		!strings.Contains(acts[0].Condition, "media_sonarr") {
+		t.Fatalf("recorded %+v (%v)", acts, err)
+	}
+	if !personsDecision(acts[0]) {
+		t.Error("dropping a dead letter is counted as a repair, so S15 would want a healer for it")
+	}
+	if _, err := askDeadLetters(t, map[string]any{"id": "1"}); err == nil {
+		t.Errorf("dead letter %d is still answered after it was dropped", d.ID)
+	}
+}
+
+// Open while DEAD_LETTERS holds a message for the consumer, in words the operator reads in one pass:
+// what is held, and where to act.
+func TestAConsumersDeadLettersAreSaidUntilActedOn(t *testing.T) {
+	f := &signalFacts{now: time.Now(), deadLetters: map[string]int{"EVENTS.media_sonarr": 4, "EVENTS.controller": 1}}
+	found := watchDeadLetters(f)
+	if len(found) != 2 {
+		t.Fatalf("said %d conditions", len(found))
+	}
+	for _, o := range found {
+		if o.Kind != "max-deliveries" || o.Severity != conditions.Warning || o.Needs == "" {
+			t.Errorf("%+v", o)
+		}
+		if why, ok := conditions.PlainWords(conditions.Words{Headline: o.Headline, Needs: o.Needs,
+			Explanation: o.Explanation, Resolved: o.Resolved}, "media"); !ok {
+			t.Errorf("%q is not plain: %s", o.Headline, why)
+		}
+		if !strings.Contains(o.Needs, "mesh MCP server") {
+			t.Errorf("does not say where to act: %q", o.Needs)
+		}
+	}
+	sonarr := found[1]
+	if sonarr.ID != "EVENTS.media_sonarr" || sonarr.Machine != "media" ||
+		sonarr.Headline != "Sonarr on media could not handle 4 messages" ||
+		!strings.Contains(sonarr.Summary, "DEAD_LETTERS") {
+		t.Errorf("%+v", sonarr)
+	}
+	if found[0].Headline != "The controller could not handle a message" {
+		t.Errorf("%q", found[0].Headline)
+	}
+	// None held, none said: it clears when they are delivered again or dropped.
+	if left := watchDeadLetters(&signalFacts{now: time.Now()}); len(left) != 0 {
+		t.Fatalf("%v", left)
+	}
+}