Age a restricted stream's dead letters out with it and never show their body (issue 728)

DEAD_LETTERS kept a message given up on MAIL forever and showed it whole to any caller of dead-letters, outside the grant and past MAIL's week. Per ADR 0376 §3-§6: the kept copy carries a per-message time to live of what is left of its stream's age (a copy already past it is kept without its body), DEAD_LETTERS and EVENTS allow per-message TTL, the list and show give no body or own headers for it, delivering it again needs a minute left and the copy keeps the time left, and its max-deliveries condition clears saying how many aged out undelivered and from which stream.

cmd/mesh-controller/deadletters_aging.go

@@ -0,0 +1,117 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// A dead letter from a restricted stream ages out with its stream (novox/hq ADR 0376 §3, §6).
+//
+// DEAD_LETTERS lets such a message go when its stream's maximum age runs out, so a consumer's `max-deliveries`
+// condition can clear with nobody having delivered it again or dropped it. That is said: the watchdogs remember,
+// from one look to the next, when each restricted dead letter ages out, count those gone once their time passed,
+// say each in the controller's log as it goes, and clear the condition with words saying how many aged out
+// undelivered and from which stream.
+
+// agingTally is what the watchdogs remember of restricted dead letters between looks.
+type agingTally struct {
+	// seen is each consumer's restricted dead letters at the last look, by id, with when each ages out.
+	seen map[string]map[uint64]time.Time
+	// agedOut is how many of each consumer's aged out undelivered while its condition was open.
+	agedOut map[string]int
+}
+
+// look compares this look's restricted dead letters with the last: newly is how many of each consumer's aged out
+// since, and cleared how many aged out in all for each consumer DEAD_LETTERS no longer holds anything of, when any
+// did. held is every consumer's count this look; aging its restricted dead letters.
+func (t *agingTally) look(now time.Time, held map[string]int, aging map[string][]link.Aging) (newly, cleared map[string]int) {
+	if t.seen == nil {
+		t.seen, t.agedOut = map[string]map[uint64]time.Time{}, map[string]int{}
+	}
+	newly, cleared = map[string]int{}, map[string]int{}
+	for key, before := range t.seen {
+		still := map[uint64]bool{}
+		for _, a := range aging[key] {
+			still[a.ID] = true
+		}
+		for id, agesOut := range before {
+			if !still[id] && !agesOut.After(now) {
+				newly[key]++
+			}
+		}
+		if newly[key] == 0 {
+			delete(newly, key)
+		}
+		t.agedOut[key] += newly[key]
+	}
+	for key, n := range t.agedOut {
+		if held[key] > 0 {
+			continue
+		}
+		if n > 0 {
+			cleared[key] = n
+		}
+		delete(t.agedOut, key)
+	}
+	t.seen = map[string]map[uint64]time.Time{}
+	for key, letters := range aging {
+		t.seen[key] = map[uint64]time.Time{}
+		for _, a := range letters {
+			t.seen[key][a.ID] = a.AgesOut
+		}
+	}
+	return newly, cleared
+}
+
+// agedOutResolved is the line a consumer's condition clears with when messages it gave up on aged out undelivered.
+func agedOutResolved(key string, n int) string {
+	stream, _, _ := strings.Cut(key, ".")
+	messages := "a message"
+	if n > 1 {
+		messages = fmt.Sprintf("%d messages", n)
+	}
+	return fmt.Sprintf("Resolved: %s of the stream %s aged out undelivered", messages, stream)
+}
+
+// sayAgedOut says, in the log, what aged out since the last look, and clears the condition of every consumer whose
+// dead letters are all gone with the words saying how many aged out. Before the rows are reconciled, so the
+// condition is cleared with these words and not its own.
+func (w *watchdogs) sayAgedOut(ctx context.Context, f *signalFacts) []string {
+	if f.advisoriesErr != nil || f.deadLettersAging == nil {
+		return nil
+	}
+	newly, cleared := w.aging.look(f.now, f.deadLetters, f.deadLettersAging)
+	var problems []string
+	for _, key := range countedKeys(newly) {
+		stream, consumer, _ := strings.Cut(key, ".")
+		fmt.Printf("%d message(s) %s gave up on aged out of %s undelivered: %s keeps them only for its maximum age "+
+			"(novox/hq ADR 0376)\n", newly[key], link.ConsumerInWords(stream, consumer), "DEAD_LETTERS", stream)
+	}
+	for _, key := range countedKeys(cleared) {
+		stream, consumer, _ := strings.Cut(key, ".")
+		fmt.Printf("%s no longer has a message kept: %d aged out of %s undelivered, so its condition clears "+
+			"(novox/hq ADR 0376)\n", link.ConsumerInWords(stream, consumer), cleared[key], stream)
+		if _, err := w.keeper.ClearSaying(ctx, conditions.Key(conditions.ScopeBus, key, link.AdvisoryMaxDeliveries),
+			fmt.Sprintf("its %d dead letter(s) of %s aged out undelivered", cleared[key], stream),
+			agedOutResolved(key, cleared[key])); err != nil {
+			problems = append(problems, err.Error())
+		}
+	}
+	return problems
+}
+
+// countedKeys is a count's keys in order.
+func countedKeys(m map[string]int) []string {
+	out := make([]string, 0, len(m))
+	for k := range m {
+		out = append(out, k)
+	}
+	sort.Strings(out)
+	return out
+}

cmd/mesh-controller/deadletters_aging_test.go

@@ -0,0 +1,69 @@
+package main
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// **A dead letter that ages out clears its condition and says so** (novox/hq ADR 0376 §6), on a fake clock: while
+// DEAD_LETTERS holds a restricted stream's messages for a consumer its max-deliveries condition is open; once they
+// have aged out it clears, with words saying how many aged out undelivered and from which stream. One dropped before
+// its age is not counted as aged out, and a consumer of EVENTS clears as before.
+func TestADeadLetterThatAgesOutClearsItsConditionAndSaysSo(t *testing.T) {
+	now := time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)
+	store := conditions.NewInMemory()
+	told := &conditions.Told{}
+	k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, Teller: told,
+		Now: func() time.Time { return now }})
+	w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
+
+	look := func(at time.Time, held map[string]int, aging map[string][]link.Aging) {
+		f := calm(at)
+		f.deadLetters, f.deadLettersAging = held, aging
+		w.see(t.Context(), f)
+	}
+	mail := "MAIL.ace_mailer"
+	look(now, map[string]int{mail: 3, "EVENTS.media_sonarr": 1}, map[string][]link.Aging{mail: {
+		{ID: 1, AgesOut: now.Add(time.Hour)}, {ID: 2, AgesOut: now.Add(2 * time.Hour)}, {ID: 3, AgesOut: now.Add(3 * time.Hour)}}})
+	if open, _ := k.Open(t.Context()); len(open) != 2 {
+		t.Fatalf("open: %+v", open)
+	}
+	// One dropped by a person before its age: not aged out.
+	look(now.Add(30*time.Minute), map[string]int{mail: 2, "EVENTS.media_sonarr": 1}, map[string][]link.Aging{mail: {
+		{ID: 2, AgesOut: now.Add(2 * time.Hour)}, {ID: 3, AgesOut: now.Add(3 * time.Hour)}}})
+	look(now.Add(150*time.Minute), map[string]int{mail: 1}, map[string][]link.Aging{mail: {{ID: 3, AgesOut: now.Add(3 * time.Hour)}}})
+	open, _ := k.Open(t.Context())
+	if len(open) != 1 || open[0].Key != conditions.Key(conditions.ScopeBus, mail, link.AdvisoryMaxDeliveries) {
+		t.Fatalf("after one aged out: %+v", open)
+	}
+	look(now.Add(4*time.Hour), map[string]int{}, map[string][]link.Aging{})
+	if open, _ := k.Open(t.Context()); len(open) != 0 {
+		t.Fatalf("after all aged out: %+v", open)
+	}
+	k.Close(context.Background()) // everything said is said once it closes
+	var mailCleared, sonarrCleared string
+	for _, e := range told.Said() {
+		if e.Change != conditions.ChangeCleared {
+			continue
+		}
+		switch e.Condition.Key {
+		case conditions.Key(conditions.ScopeBus, mail, link.AdvisoryMaxDeliveries):
+			mailCleared = e.Condition.Resolved
+		case conditions.Key(conditions.ScopeBus, "EVENTS.media_sonarr", link.AdvisoryMaxDeliveries):
+			sonarrCleared = e.Condition.Resolved
+		}
+	}
+	if mailCleared != "Resolved: 2 messages of the stream MAIL aged out undelivered" {
+		t.Fatalf("MAIL's condition cleared with %q", mailCleared)
+	}
+	if sonarrCleared != "Resolved: the messages it gave up on were delivered again or dropped" {
+		t.Fatalf("an EVENTS consumer's condition cleared with %q", sonarrCleared)
+	}
+	if why, ok := conditions.Plain(mailCleared); !ok {
+		t.Fatalf("not plain: %s", why)
+	}
+}

cmd/mesh-controller/probes.go

@@ -929,6 +929,12 @@ func streamDiffers(want broker.Stream, have nats.StreamConfig) string {
 		differs = append(differs, fmt.Sprintf("keeps one of a message id for %s, defined %s", have.Duplicates,
 			time.Duration(want.DuplicatesSeconds)*time.Second))
 	}
+	if want.AllowMsgTTL && !have.AllowMsgTTL {
+		differs = append(differs, "takes no time to live of a message's own, defined to (novox/hq ADR 0376)")
+	}
+	if want.Restricted && have.Metadata[broker.RestrictedMetadata] != "true" {
+		differs = append(differs, "does not say it is restricted, so a message kept from it would not age out with it")
+	}
 	if want.DiscardNew && have.Discard != nats.DiscardNew {
 		differs = append(differs, "drops what it holds when full, defined to refuse what comes next")
 	}

cmd/mesh-controller/watchdogs.go

@@ -91,7 +91,10 @@ type signalFacts struct {
 	// deadLettersNewest is when DEAD_LETTERS kept the newest message it holds for each of those
 	// consumers: the condition counts the messages given up on, not the looks (novox/hq issue 440).
 	deadLettersNewest map[string]time.Time
-	advisoriesErr     error
+	// deadLettersAging is, for each consumer whose dead letters came from a restricted stream, when each ages out
+	// (novox/hq ADR 0376 §6); nil when it was not read.
+	deadLettersAging map[string][]link.Aging
+	advisoriesErr    error
 	// busFaultsErr is why the bus module's events could not be read this tick (novox/hq ADR 0357).
 	busFaultsErr error
 
@@ -211,6 +214,9 @@ type watchdogs struct {
 	// confirm holds back a row blind for one tick: a read that timed out once on a loaded store.
 	confirm confirming
 
+	// aging is what is remembered of restricted dead letters between looks (novox/hq ADR 0376 §6).
+	aging agingTally
+
 	mu     sync.Mutex
 	ticked time.Time
 	last   *signalFacts
@@ -264,6 +270,8 @@ func (w *watchdogs) tick(ctx context.Context) {
 func (w *watchdogs) see(running context.Context, f *signalFacts) {
 	var problems []string
 	var blind []conditions.Observation
+	// What aged out of DEAD_LETTERS, said and cleared in its own words before the rows clear it in theirs.
+	problems = append(problems, w.sayAgedOut(running, f)...)
 	for _, row := range watchedRows() {
 		if err := row.needs(f); err != nil {
 			blind = append(blind, blindRow(row, err))
@@ -382,6 +390,9 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts {
 			}
 		}
 	}
+	if f.advisoriesErr == nil && w.js != nil {
+		f.deadLettersAging, f.advisoriesErr = link.AgingDeadLetters(w.js.Context(), f.deadLetters)
+	}
 	f.handActs, f.handActsErr = w.gatherHandActs(ctx, now)
 	f.refusedSends, f.refusedSendsErr = inv.RefusedSendsSince(ctx, now.Add(-generationRefusalsSaid))
 	f.doing, f.doingErr = inv.NotDoingWhatTheyWereTold(ctx)