Count a dead letter by when it was kept, so one kept late still counts (issue 440)
mesh/merge-gate pass: builds build-agent, mesh-controller → ace, g14, novox, shanks; no bus step; every machine composes with the change as it did without …
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery delivered

The give-up time is not newer for every letter kept: a notice that could
not be kept is offered again a minute later, so a later give-up can be kept
first and the one after it read as not fresh. The stored time rises with the
stream's sequence. A consumer whose letters vanish between the two reads is
left out of the look instead of counted as a look, and the skip of a
token-less max-deliveries advisory now has a test of its own.
This commit is contained in:
2026-10-11 02:36:58 +02:00
parent f83fcdc15f
commit cfbbad8209
6 changed files with 74 additions and 35 deletions
+8 -9
View File
@@ -167,10 +167,13 @@ func HeldDeadLetters(js nats.JetStreamContext) (map[string]int, error) {
return held, nil
}
// NewestDeadLetters is when each consumer of held last gave up on a message DEAD_LETTERS still holds, by
// `<stream>.<consumer>`: the newest kept, by when the server said it gave up, or when it was kept if that
// was not said. The consumer's condition counts the messages given up on by it, never how often the
// controller looked at them (novox/hq issue 440).
// NewestDeadLetters is when DEAD_LETTERS kept the newest message it holds for each consumer of held, by
// `<stream>.<consumer>`: the stored time of its last message, which rises with the stream's sequence. A
// consumer whose letters were all delivered again or dropped since held was read is left out. The
// consumer's condition counts the messages given up on by it, never how often the controller looked at
// them (novox/hq issue 440), so it needs a time that is newer for every message newly kept. The time the
// server said it gave up is not one: a notice that could not be kept is offered again a minute later
// (noticeRetry), so a later give-up can be kept first, and the one kept after it carries an older time.
func NewestDeadLetters(js nats.JetStreamContext, held map[string]int) (map[string]time.Time, error) {
newest := map[string]time.Time{}
for key := range held {
@@ -182,11 +185,7 @@ func NewestDeadLetters(js nats.JetStreamContext, held map[string]int) (map[strin
if err != nil {
return nil, fmt.Errorf("the newest dead letter of %s cannot be read: %w", key, err)
}
at := deadLetterOf(raw.Sequence, raw.Subject, raw.Header, nil, false).GaveUp
if at.IsZero() {
at = raw.Time
}
newest[key] = at.UTC()
newest[key] = raw.Time.UTC()
}
return newest, nil
}
+37 -20
View File
@@ -335,32 +335,49 @@ func TestNoticesThatCannotBeTakenAreSaidAndServingGoesOn(t *testing.T) {
eventually(t, "a report heard while the notices cannot be taken", func() bool { return held.count() == 1 })
}
// When each consumer last gave up on a message DEAD_LETTERS still holds, for the condition that counts
// what was given up on rather than how often it was looked at (novox/hq issue 440): the newest kept, by
// when the server said it gave up.
func TestTheNewestHeldDeadLetterSaysWhenItWasGivenUp(t *testing.T) {
// When DEAD_LETTERS kept each consumer's newest message, for the condition that counts what was given up
// on rather than how often it was looked at (novox/hq issue 440). It rises with every message kept, even
// one the consumer gave up on before the last: a notice that could not be kept is offered again a minute
// later, so a later give-up can be kept first, and the one kept after it must still count.
func TestTheNewestHeldDeadLetterIsWhenItWasKept(t *testing.T) {
js := aBus(t)
first := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC)
for seq, at := range map[int]time.Time{1: first, 2: first.Add(time.Minute)} {
if _, err := KeepDeadLetter(js.Context(), []byte(fmt.Sprintf(`{"stream":"EVENTS","consumer":"media_sonarr",`+
`"stream_seq":%d,"deliveries":5,"timestamp":%q}`, seq, at.Format(time.RFC3339Nano)))); err != nil {
gaveUp := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC)
keep := func(consumer string, seq int, at time.Time) {
t.Helper()
if _, err := KeepDeadLetter(js.Context(), []byte(fmt.Sprintf(`{"stream":"EVENTS","consumer":%q,`+
`"stream_seq":%d,"deliveries":5,"timestamp":%q}`, consumer, seq, at.Format(time.RFC3339Nano)))); err != nil {
t.Fatal(err)
}
}
if _, err := KeepDeadLetter(js.Context(), []byte(fmt.Sprintf(`{"stream":"EVENTS","consumer":"media_radarr",`+
`"stream_seq":3,"deliveries":5,"timestamp":%q}`, first.Format(time.RFC3339Nano)))); err != nil {
t.Fatal(err)
look := func() map[string]time.Time {
t.Helper()
held, err := HeldDeadLetters(js.Context())
if err != nil {
t.Fatal(err)
}
newest, err := NewestDeadLetters(js.Context(), held)
if err != nil {
t.Fatal(err)
}
return newest
}
held, err := HeldDeadLetters(js.Context())
if err != nil {
t.Fatal(err)
// Sonarr gave up at 21:05 and that is kept first; radarr's letter is kept after it.
keep("media_sonarr", 2, gaveUp.Add(time.Minute))
keep("media_radarr", 3, gaveUp)
before := look()
if len(before) != 2 || before["EVENTS.media_sonarr"].IsZero() ||
!before["EVENTS.media_radarr"].After(before["EVENTS.media_sonarr"]) {
t.Fatalf("%v", before)
}
newest, err := NewestDeadLetters(js.Context(), held)
if err != nil {
t.Fatal(err)
// Sonarr's give-up of 21:04, kept late, is newer than anything the condition has seen.
keep("media_sonarr", 1, gaveUp)
after := look()
if !after["EVENTS.media_sonarr"].After(before["EVENTS.media_sonarr"]) ||
!after["EVENTS.media_sonarr"].After(before["EVENTS.media_radarr"]) {
t.Fatalf("a letter kept late does not read as newer: before %v, after %v", before, after)
}
if len(newest) != 2 || !newest["EVENTS.media_sonarr"].Equal(first.Add(time.Minute)) ||
!newest["EVENTS.media_radarr"].Equal(first) {
t.Fatalf("%v", newest)
if !after["EVENTS.media_radarr"].Equal(before["EVENTS.media_radarr"]) {
t.Fatalf("radarr's newest moved: before %v, after %v", before, after)
}
}