Give a consumer's max-deliveries key one watcher, counting what was given up (issue 440)
mesh/merge-gate pass: builds build-agent, mesh-controller → ace, g14, novox, shanks; no bus step; every machine composes with the change as it did without …
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery superseded: a newer head of the same pull request

The dead-letter row and a max-deliveries advisory without a token both said
bus.<stream>.<consumer>.max-deliveries: each look added an observation and a
line of evidence through the row, and the two overwrote each other's words.
The row owns the key since issue 330, so the advisory watcher leaves it, and
the row now says when the newest held message was given up, so a letter held
for a day no longer reads as observed every 30 seconds. Times without Happened
is refused, since the raise ignored it and an update counted it.
This commit is contained in:
2026-10-11 02:30:09 +02:00
parent f8d08b0637
commit f83fcdc15f
8 changed files with 176 additions and 6 deletions
+24
View File
@@ -167,6 +167,30 @@ func HeldDeadLetters(js nats.JetStreamContext) (map[string]int, error) {
return held, nil
}
// NewestDeadLetters is when each consumer of held last gave up on a message DEAD_LETTERS still holds, by
// `<stream>.<consumer>`: the newest kept, by when the server said it gave up, or when it was kept if that
// was not said. The consumer's condition counts the messages given up on by it, never how often the
// controller looked at them (novox/hq issue 440).
func NewestDeadLetters(js nats.JetStreamContext, held map[string]int) (map[string]time.Time, error) {
newest := map[string]time.Time{}
for key := range held {
stream, consumer, _ := strings.Cut(key, ".")
raw, err := js.GetLastMsg(broker.DeadLettersStream, broker.DeadLetterSubject(stream, consumer))
if errors.Is(err, nats.ErrMsgNotFound) {
continue // delivered again or dropped since it was counted
}
if err != nil {
return nil, fmt.Errorf("the newest dead letter of %s cannot be read: %w", key, err)
}
at := deadLetterOf(raw.Sequence, raw.Subject, raw.Header, nil, false).GaveUp
if at.IsZero() {
at = raw.Time
}
newest[key] = at.UTC()
}
return newest, nil
}
// DeadLetters lists what DEAD_LETTERS holds, newest first, at most most of them (all when most is not
// positive); for one consumer when consumer names one (its name, or `<stream>.<consumer>`). The total is
// the stream's own count per consumer, so it is right however few are read.
+30
View File
@@ -334,3 +334,33 @@ func TestNoticesThatCannotBeTakenAreSaidAndServingGoesOn(t *testing.T) {
}
eventually(t, "a report heard while the notices cannot be taken", func() bool { return held.count() == 1 })
}
// When each consumer last gave up on a message DEAD_LETTERS still holds, for the condition that counts
// what was given up on rather than how often it was looked at (novox/hq issue 440): the newest kept, by
// when the server said it gave up.
func TestTheNewestHeldDeadLetterSaysWhenItWasGivenUp(t *testing.T) {
js := aBus(t)
first := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC)
for seq, at := range map[int]time.Time{1: first, 2: first.Add(time.Minute)} {
if _, err := KeepDeadLetter(js.Context(), []byte(fmt.Sprintf(`{"stream":"EVENTS","consumer":"media_sonarr",`+
`"stream_seq":%d,"deliveries":5,"timestamp":%q}`, seq, at.Format(time.RFC3339Nano)))); err != nil {
t.Fatal(err)
}
}
if _, err := KeepDeadLetter(js.Context(), []byte(fmt.Sprintf(`{"stream":"EVENTS","consumer":"media_radarr",`+
`"stream_seq":3,"deliveries":5,"timestamp":%q}`, first.Format(time.RFC3339Nano)))); err != nil {
t.Fatal(err)
}
held, err := HeldDeadLetters(js.Context())
if err != nil {
t.Fatal(err)
}
newest, err := NewestDeadLetters(js.Context(), held)
if err != nil {
t.Fatal(err)
}
if len(newest) != 2 || !newest["EVENTS.media_sonarr"].Equal(first.Add(time.Minute)) ||
!newest["EVENTS.media_radarr"].Equal(first) {
t.Fatalf("%v", newest)
}
}