Merge pull request 'A consumer's max-deliveries condition has one watcher and counts what was given up (issue 440)' (#214) from fix/440-one-key-one-watcher into main

cmd/mesh-controller/advisories_counted_test.go

@@ -93,3 +93,88 @@ func TestAConditionOfThePresentCountsItsLooks(t *testing.T) {
 		t.Fatalf("observed %d time(s), %d evidence lines, last %s", c.Observations, len(c.Evidence), c.LastObserved)
 	}
 }
+
+// **One key, one watcher** (novox/hq issue 440). A consumer's max-deliveries condition is said from
+// DEAD_LETTERS while it holds a message the consumer gave up on (issue 330). A max-deliveries advisory
+// without a token names the same key; read beside it, each look added an observation and a line of
+// evidence through the dead-letter half, and the two summaries overwrote each other on every look. The
+// dead-letter row alone says that key, and counts the messages given up on, never the looks.
+func TestAHeldDeadLetterIsCountedByWhatWasGivenUpNotByTheLooks(t *testing.T) {
+	gaveUp := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC)
+	now := gaveUp.Add(20 * time.Second)
+	store := conditions.NewInMemory()
+	k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
+		Teller: &conditions.Told{}, Now: func() time.Time { return now }})
+	t.Cleanup(func() { k.Close(t.Context()) })
+
+	const key = "bus.EVENTS.media_sonarr.max-deliveries"
+	held := map[string]int{"EVENTS.media_sonarr": 1}
+	newest := map[string]time.Time{"EVENTS.media_sonarr": gaveUp}
+	advisory := link.Advisory{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.media_sonarr", Stream: "EVENTS",
+		Consumer: "media_sonarr", Said: "sonarr on media handed message 7 over 5 times and gave up on it",
+		First: gaveUp, Last: gaveUp, Count: 1}
+	look := func() conditions.Condition {
+		t.Helper()
+		f := &signalFacts{now: now, host: "novox", advisories: []link.Advisory{advisory},
+			deadLetters: held, deadLettersNewest: newest}
+		if err := k.Reconcile(t.Context(), "S9", watchAdvisories(f)); err != nil {
+			t.Fatal(err)
+		}
+		c, found, err := conditions.ReadOne(t.Context(), store, key)
+		if err != nil || !found {
+			t.Fatalf("%s: found %v, %v", key, found, err)
+		}
+		if !strings.Contains(c.Summary, "dead-letters verb") {
+			t.Fatalf("the summary is not the dead-letter row's: %q", c.Summary)
+		}
+		return c
+	}
+
+	var c conditions.Condition
+	for range 5 {
+		c = look()
+		now = now.Add(30 * time.Second)
+	}
+	if c.Observations != 1 || len(c.Evidence) != 1 || !c.LastObserved.Equal(gaveUp) {
+		t.Fatalf("one message given up on, looked at five times, reads as observed %d time(s), last %s, "+
+			"with %d evidence lines: %+v", c.Observations, c.LastObserved, len(c.Evidence), c.Evidence)
+	}
+
+	// The consumer gives up on a second message between two looks: two given up on, one more line.
+	again := now.Add(-10 * time.Second)
+	held["EVENTS.media_sonarr"], newest["EVENTS.media_sonarr"] = 2, again
+	c = look()
+	if c.Observations != 2 || len(c.Evidence) != 2 || !c.LastObserved.Equal(again) {
+		t.Fatalf("two given up on read as observed %d time(s), last %s, evidence %+v", c.Observations,
+			c.LastObserved, c.Evidence)
+	}
+	now = now.Add(30 * time.Second)
+	if c = look(); c.Observations != 2 || len(c.Evidence) != 2 {
+		t.Fatalf("a look with nothing new counted: observed %d time(s), evidence %+v", c.Observations, c.Evidence)
+	}
+}
+
+// The advisory still says the max-deliveries it alone knows: a message given up on that could not be
+// kept, under a key of its own (issue 330), which issue 440's change leaves as it was.
+func TestAMessageThatCouldNotBeKeptIsStillSaidFromTheAdvisory(t *testing.T) {
+	at := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC)
+	f := &signalFacts{now: at, host: "novox", advisories: []link.Advisory{{Kind: link.AdvisoryMaxDeliveries,
+		ID: "EVENTS.media_sonarr", Stream: "EVENTS", Consumer: "media_sonarr", Token: link.AdvisoryNotKept,
+		Said: "sonarr on media gave up on message 7, and it could not be kept", First: at, Last: at, Count: 1}}}
+	said := watchAdvisories(f)
+	if len(said) != 1 || said[0].Key() != "bus.EVENTS.media_sonarr.not-kept" {
+		t.Fatalf("%+v", said)
+	}
+}
+
+// A max-deliveries advisory without a token names a consumer's dead-letter key, which only DEAD_LETTERS
+// says (novox/hq issues 330 and 440): with nothing held, the advisory alone raises nothing.
+func TestAMaxDeliveriesAdvisoryAloneRaisesNothing(t *testing.T) {
+	at := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC)
+	f := &signalFacts{now: at, host: "novox", advisories: []link.Advisory{{Kind: link.AdvisoryMaxDeliveries,
+		ID: "EVENTS.media_sonarr", Stream: "EVENTS", Consumer: "media_sonarr",
+		Said: "sonarr on media handed message 7 over 5 times and gave up on it", First: at, Last: at, Count: 1}}}
+	if said := watchAdvisories(f); len(said) != 0 {
+		t.Fatalf("said %+v", said)
+	}
+}

cmd/mesh-controller/signals.go

@@ -523,6 +523,14 @@ func watchAdvisories(f *signalFacts) []conditions.Observation {
 		if a.Kind == link.AdvisoryConsumerLost && !f.lostConsumers[a.Stream+"."+a.Consumer] {
 			continue // it exists again, or the mesh no longer expects it: a removal, not a loss
 		}
+		if a.Kind == link.AdvisoryMaxDeliveries && a.Token == "" {
+			// A consumer's max-deliveries key is the dead-letter row's (novox/hq issue 330): said while
+			// DEAD_LETTERS holds what it gave up on, cleared when that is delivered again or dropped. The
+			// listener records no such advisory today; one read here as well added a look to the count and
+			// overwrote the row's words on every look (novox/hq issue 440). Only one that could not be kept
+			// (AdvisoryNotKept), which DEAD_LETTERS cannot say, is said from here.
+			continue
+		}
 		severity := conditions.Warning
 		times := ""
 		if a.Count > 1 {
@@ -568,7 +576,17 @@ func watchDeadLetters(f *signalFacts) []conditions.Observation {
 			messages, them = fmt.Sprintf("%d messages", n), "them"
 		}
 		who := consumerWho(stream, consumer)
+		// DEAD_LETTERS is a record of what was given up on: the condition says when the newest held message
+		// was kept, and its count is a running total of the messages given up on while it is open, never
+		// how often the controller looked (novox/hq issues 402 and 440). It is at least the number held now,
+		// and does not go down when one is delivered again or dropped. Without that time it counts its
+		// looks, as a source of the present does.
+		happened, times := f.deadLettersNewest[key], 0
+		if !happened.IsZero() {
+			times = n
+		}
 		out = append(out, conditions.Observation{Scope: conditions.ScopeBus, ID: key, Kind: link.AdvisoryMaxDeliveries,
+			Happened: happened, Times: times,
 			Machine: consumerMachine(stream, consumer), Severity: conditions.Warning,
 			Summary: fmt.Sprintf("%s gave up on %s; %s kept in %s until delivered again or dropped, with why, "+
 				"through the controller's dead-letters verb", link.ConsumerInWords(stream, consumer), messages,

cmd/mesh-controller/signals_test.go

@@ -100,14 +100,15 @@ var suppressions = map[string]suppression{
 		inside: func(f *signalFacts) { f.standings = []conditions.Condition{standingSaid(f.now.Add(-29 * time.Minute))} },
 		past:   func(f *signalFacts) { f.standings = []conditions.Condition{standingSaid(f.now.Add(-31 * time.Minute))} },
 	},
+	// A message given up on and not kept: the one max-deliveries advisory S9 says itself (issue 440).
 	"S9": {
 		inside: func(f *signalFacts) {
-			f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop", Said: "gave up",
-				First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-61 * time.Minute), Count: 1}}
+			f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop",
+				Token: link.AdvisoryNotKept, Said: "gave up, not kept", First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-61 * time.Minute), Count: 1}}
 		},
 		past: func(f *signalFacts) {
-			f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop", Said: "gave up",
-				First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-59 * time.Minute), Count: 1}}
+			f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop",
+				Token: link.AdvisoryNotKept, Said: "gave up, not kept", First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-59 * time.Minute), Count: 1}}
 		},
 	},
 	"S10": {

cmd/mesh-controller/watchdogs.go

@@ -79,8 +79,11 @@ type signalFacts struct {
 	lostConsumers map[string]bool
 	// deadLetters are how many messages DEAD_LETTERS holds per consumer, by `<stream>.<consumer>`
 	// (novox/hq issue 330): each consumer's max-deliveries condition is open while it holds any.
-	deadLetters   map[string]int
-	advisoriesErr error
+	deadLetters map[string]int
+	// deadLettersNewest is when DEAD_LETTERS kept the newest message it holds for each of those
+	// consumers: the condition counts the messages given up on, not the looks (novox/hq issue 440).
+	deadLettersNewest map[string]time.Time
+	advisoriesErr     error
 
 	selfCheck selfCheckFacts
 
@@ -350,6 +353,16 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts {
 	if f.advisoriesErr == nil && w.js != nil {
 		f.deadLetters, f.advisoriesErr = link.HeldDeadLetters(w.js.Context())
 	}
+	if f.advisoriesErr == nil && len(f.deadLetters) > 0 {
+		f.deadLettersNewest, f.advisoriesErr = link.NewestDeadLetters(w.js.Context(), f.deadLetters)
+		for key := range f.deadLetters {
+			if _, ok := f.deadLettersNewest[key]; !ok && f.advisoriesErr == nil {
+				// Delivered again or dropped between the two reads: left out of this look, rather than
+				// observed without a time and counted as a look (novox/hq issue 440).
+				delete(f.deadLetters, key)
+			}
+		}
+	}
 	f.handActs, f.handActsErr = w.gatherHandActs(ctx, now)
 	f.facts.taken, _, f.facts.began, f.facts.err = exportedFacts.last()
 	return f