Give a consumer's max-deliveries key one watcher, counting what was given up (issue 440)

The dead-letter row and a max-deliveries advisory without a token both said
bus.<stream>.<consumer>.max-deliveries: each look added an observation and a
line of evidence through the row, and the two overwrote each other's words.
The row owns the key since issue 330, so the advisory watcher leaves it, and
the row now says when the newest held message was given up, so a letter held
for a day no longer reads as observed every 30 seconds. Times without Happened
is refused, since the raise ignored it and an update counted it.

internal/conditions/condition.go

@@ -276,6 +276,10 @@ func (o Observation) check() error {
 		return fmt.Errorf("the condition %s says nothing", o.Key())
 	case strings.TrimSpace(o.Source) == "":
 		return fmt.Errorf("the condition %s does not say what raised it", o.Key())
+	case o.Times != 0 && o.Happened.IsZero():
+		// Times is what a record counts beside when it last happened: alone, the raise would ignore it
+		// and an update count it, so the count would mean two things (novox/hq issue 440).
+		return fmt.Errorf("the condition %s says how often it happened (%d) but not when", o.Key(), o.Times)
 	}
 	return nil
 }

internal/conditions/store_test.go

@@ -494,3 +494,19 @@ func TestASecondReopeningKeepsBothGaps(t *testing.T) {
 		t.Fatalf("open at the wrong moments: %+v", g)
 	}
 }
+
+// **Times is how often a record says it happened, never alone** (novox/hq issue 440). The raise read
+// Times only beside Happened while an update read it always, so a source setting Times alone would have
+// jumped the count on its second look and not its first. The keeper refuses it, saying why.
+func TestTimesWithoutWhenItHappenedIsRefused(t *testing.T) {
+	k, store, _, _ := keeper(t)
+	o := Observation{Scope: ScopeMachine, ID: "ace", Kind: "silent", Machine: "ace", Severity: Warning,
+		Summary: "ace has not been heard from", Source: "S1", Times: 5}
+	_, err := k.Observe(t.Context(), o)
+	if err == nil || !strings.Contains(err.Error(), "when") {
+		t.Fatalf("Times without Happened was taken: %v", err)
+	}
+	if _, found, _ := ReadOne(t.Context(), store, o.Key()); found {
+		t.Fatal("a refused observation raised its condition")
+	}
+}

internal/link/deadletters.go

@@ -167,6 +167,30 @@ func HeldDeadLetters(js nats.JetStreamContext) (map[string]int, error) {
 	return held, nil
 }
 
+// NewestDeadLetters is when each consumer of held last gave up on a message DEAD_LETTERS still holds, by
+// `<stream>.<consumer>`: the newest kept, by when the server said it gave up, or when it was kept if that
+// was not said. The consumer's condition counts the messages given up on by it, never how often the
+// controller looked at them (novox/hq issue 440).
+func NewestDeadLetters(js nats.JetStreamContext, held map[string]int) (map[string]time.Time, error) {
+	newest := map[string]time.Time{}
+	for key := range held {
+		stream, consumer, _ := strings.Cut(key, ".")
+		raw, err := js.GetLastMsg(broker.DeadLettersStream, broker.DeadLetterSubject(stream, consumer))
+		if errors.Is(err, nats.ErrMsgNotFound) {
+			continue // delivered again or dropped since it was counted
+		}
+		if err != nil {
+			return nil, fmt.Errorf("the newest dead letter of %s cannot be read: %w", key, err)
+		}
+		at := deadLetterOf(raw.Sequence, raw.Subject, raw.Header, nil, false).GaveUp
+		if at.IsZero() {
+			at = raw.Time
+		}
+		newest[key] = at.UTC()
+	}
+	return newest, nil
+}
+
 // DeadLetters lists what DEAD_LETTERS holds, newest first, at most most of them (all when most is not
 // positive); for one consumer when consumer names one (its name, or `<stream>.<consumer>`). The total is
 // the stream's own count per consumer, so it is right however few are read.

internal/link/deadletters_test.go

@@ -334,3 +334,33 @@ func TestNoticesThatCannotBeTakenAreSaidAndServingGoesOn(t *testing.T) {
 	}
 	eventually(t, "a report heard while the notices cannot be taken", func() bool { return held.count() == 1 })
 }
+
+// When each consumer last gave up on a message DEAD_LETTERS still holds, for the condition that counts
+// what was given up on rather than how often it was looked at (novox/hq issue 440): the newest kept, by
+// when the server said it gave up.
+func TestTheNewestHeldDeadLetterSaysWhenItWasGivenUp(t *testing.T) {
+	js := aBus(t)
+	first := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC)
+	for seq, at := range map[int]time.Time{1: first, 2: first.Add(time.Minute)} {
+		if _, err := KeepDeadLetter(js.Context(), []byte(fmt.Sprintf(`{"stream":"EVENTS","consumer":"media_sonarr",`+
+			`"stream_seq":%d,"deliveries":5,"timestamp":%q}`, seq, at.Format(time.RFC3339Nano)))); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if _, err := KeepDeadLetter(js.Context(), []byte(fmt.Sprintf(`{"stream":"EVENTS","consumer":"media_radarr",`+
+		`"stream_seq":3,"deliveries":5,"timestamp":%q}`, first.Format(time.RFC3339Nano)))); err != nil {
+		t.Fatal(err)
+	}
+	held, err := HeldDeadLetters(js.Context())
+	if err != nil {
+		t.Fatal(err)
+	}
+	newest, err := NewestDeadLetters(js.Context(), held)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(newest) != 2 || !newest["EVENTS.media_sonarr"].Equal(first.Add(time.Minute)) ||
+		!newest["EVENTS.media_radarr"].Equal(first) {
+		t.Fatalf("%v", newest)
+	}
+}