Merge pull request 'A consumer's max-deliveries condition has one watcher and counts what was given up (issue 440)' (#214) from fix/440-one-key-one-watcher into main

internal/conditions/condition.go

@@ -154,7 +154,9 @@ type Condition struct {
 	Gaps  []Gap     `json:"gaps,omitempty"`
 	// Observations is how many times it was observed since raised: a look that sees it, for a source
 	// that looks at the present; for one that reads a record of what happened (Observation.Happened),
-	// how many times it happened, and LastObserved when it last did (novox/hq issue 402).
+	// how many times it happened, and LastObserved when it last did (novox/hq issue 402). That count is a
+	// running total since raised, never lowered: a consumer's dead letters count each message given up on
+	// while the condition is open, not the number held now (novox/hq issue 440).
 	Observations int `json:"observations"`
 	// Count is how many times it has been raised, a reopening within ReopenWithin counted.
 	Count    int       `json:"count"`
@@ -276,6 +278,10 @@ func (o Observation) check() error {
 		return fmt.Errorf("the condition %s says nothing", o.Key())
 	case strings.TrimSpace(o.Source) == "":
 		return fmt.Errorf("the condition %s does not say what raised it", o.Key())
+	case o.Times != 0 && o.Happened.IsZero():
+		// Times is what a record counts beside when it last happened: alone, the raise would ignore it
+		// and an update count it, so the count would mean two things (novox/hq issue 440).
+		return fmt.Errorf("the condition %s says how often it happened (%d) but not when", o.Key(), o.Times)
 	}
 	return nil
 }

internal/conditions/store_test.go

@@ -494,3 +494,19 @@ func TestASecondReopeningKeepsBothGaps(t *testing.T) {
 		t.Fatalf("open at the wrong moments: %+v", g)
 	}
 }
+
+// **Times is how often a record says it happened, never alone** (novox/hq issue 440). The raise read
+// Times only beside Happened while an update read it always, so a source setting Times alone would have
+// jumped the count on its second look and not its first. The keeper refuses it, saying why.
+func TestTimesWithoutWhenItHappenedIsRefused(t *testing.T) {
+	k, store, _, _ := keeper(t)
+	o := Observation{Scope: ScopeMachine, ID: "ace", Kind: "silent", Machine: "ace", Severity: Warning,
+		Summary: "ace has not been heard from", Source: "S1", Times: 5}
+	_, err := k.Observe(t.Context(), o)
+	if err == nil || !strings.Contains(err.Error(), "when") {
+		t.Fatalf("Times without Happened was taken: %v", err)
+	}
+	if _, found, _ := ReadOne(t.Context(), store, o.Key()); found {
+		t.Fatal("a refused observation raised its condition")
+	}
+}

internal/link/deadletters.go

@@ -167,6 +167,29 @@ func HeldDeadLetters(js nats.JetStreamContext) (map[string]int, error) {
 	return held, nil
 }
 
+// NewestDeadLetters is when DEAD_LETTERS kept the newest message it holds for each consumer of held, by
+// `<stream>.<consumer>`: the stored time of its last message, which rises with the stream's sequence. A
+// consumer whose letters were all delivered again or dropped since held was read is left out. The
+// consumer's condition counts the messages given up on by it, never how often the controller looked at
+// them (novox/hq issue 440), so it needs a time that is newer for every message newly kept. The time the
+// server said it gave up is not one: a notice that could not be kept is offered again a minute later
+// (noticeRetry), so a later give-up can be kept first, and the one kept after it carries an older time.
+func NewestDeadLetters(js nats.JetStreamContext, held map[string]int) (map[string]time.Time, error) {
+	newest := map[string]time.Time{}
+	for key := range held {
+		stream, consumer, _ := strings.Cut(key, ".")
+		raw, err := js.GetLastMsg(broker.DeadLettersStream, broker.DeadLetterSubject(stream, consumer))
+		if errors.Is(err, nats.ErrMsgNotFound) {
+			continue // delivered again or dropped since it was counted
+		}
+		if err != nil {
+			return nil, fmt.Errorf("the newest dead letter of %s cannot be read: %w", key, err)
+		}
+		newest[key] = raw.Time.UTC()
+	}
+	return newest, nil
+}
+
 // DeadLetters lists what DEAD_LETTERS holds, newest first, at most most of them (all when most is not
 // positive); for one consumer when consumer names one (its name, or `<stream>.<consumer>`). The total is
 // the stream's own count per consumer, so it is right however few are read.

internal/link/deadletters_test.go

@@ -334,3 +334,50 @@ func TestNoticesThatCannotBeTakenAreSaidAndServingGoesOn(t *testing.T) {
 	}
 	eventually(t, "a report heard while the notices cannot be taken", func() bool { return held.count() == 1 })
 }
+
+// When DEAD_LETTERS kept each consumer's newest message, for the condition that counts what was given up
+// on rather than how often it was looked at (novox/hq issue 440). It rises with every message kept, even
+// one the consumer gave up on before the last: a notice that could not be kept is offered again a minute
+// later, so a later give-up can be kept first, and the one kept after it must still count.
+func TestTheNewestHeldDeadLetterIsWhenItWasKept(t *testing.T) {
+	js := aBus(t)
+	gaveUp := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC)
+	keep := func(consumer string, seq int, at time.Time) {
+		t.Helper()
+		if _, err := KeepDeadLetter(js.Context(), []byte(fmt.Sprintf(`{"stream":"EVENTS","consumer":%q,`+
+			`"stream_seq":%d,"deliveries":5,"timestamp":%q}`, consumer, seq, at.Format(time.RFC3339Nano)))); err != nil {
+			t.Fatal(err)
+		}
+	}
+	look := func() map[string]time.Time {
+		t.Helper()
+		held, err := HeldDeadLetters(js.Context())
+		if err != nil {
+			t.Fatal(err)
+		}
+		newest, err := NewestDeadLetters(js.Context(), held)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return newest
+	}
+
+	// Sonarr gave up at 21:05 and that is kept first; radarr's letter is kept after it.
+	keep("media_sonarr", 2, gaveUp.Add(time.Minute))
+	keep("media_radarr", 3, gaveUp)
+	before := look()
+	if len(before) != 2 || before["EVENTS.media_sonarr"].IsZero() ||
+		!before["EVENTS.media_radarr"].After(before["EVENTS.media_sonarr"]) {
+		t.Fatalf("%v", before)
+	}
+	// Sonarr's give-up of 21:04, kept late, is newer than anything the condition has seen.
+	keep("media_sonarr", 1, gaveUp)
+	after := look()
+	if !after["EVENTS.media_sonarr"].After(before["EVENTS.media_sonarr"]) ||
+		!after["EVENTS.media_sonarr"].After(before["EVENTS.media_radarr"]) {
+		t.Fatalf("a letter kept late does not read as newer: before %v, after %v", before, after)
+	}
+	if !after["EVENTS.media_radarr"].Equal(before["EVENTS.media_radarr"]) {
+		t.Fatalf("radarr's newest moved: before %v, after %v", before, after)
+	}
+}