diff --git a/modules/mesh-grants/grants_test.go b/modules/mesh-grants/grants_test.go index 5ae14e1d8..975123922 100644 --- a/modules/mesh-grants/grants_test.go +++ b/modules/mesh-grants/grants_test.go @@ -4167,12 +4167,14 @@ func TestGrants(t *testing.T) { } // A held consumer that differs is never changed by a pass: it is raised as a condition naming it, and brought to // the composed configuration only by hand (grants_consumer_match), with no lease key too. - var said, behind []oc.Condition + var said, behind, orphaned []oc.Condition p.Say = func(c oc.Condition) error { switch c.Key { case grants.GrantMissingCondition, grants.RefusedCondition, grants.HeldModuleCondition, grants.FullCondition: // their own rows case grants.ConsumerBehindCondition: behind = append(behind, c) + case grants.DurableOrphanedCondition: + orphaned = append(orphaned, c) default: said = append(said, c) } @@ -4222,6 +4224,32 @@ func TestGrants(t *testing.T) { if len(res.Consumers.Behind) != 0 || behind[len(behind)-1].State != occ.StateCleared { t.Errorf("a consumer back within the bound: %+v, said %+v", res.Consumers, behind) } + // Design 47 H2, H-A-472 (2): a durable far behind with no pulls waiting, whose twin on the stream (b_mod-c_tools) has + // pulls waiting and delivered within the day, is an orphan: left out of the behind check, named once in + // mesh-grants.durable-orphaned, and cleared when it is gone; with its twin not current, it is behind again. + var twin map[string]any + for _, c := range b.consumers["EVENTS"] { + if c["name"] == "b_mod-c_tools" { + twin = c + } + } + twin["waiting_pulls"], twin["last_delivered"] = 1, "2026-10-25T21:00:00.5Z" + made["pending"] = grants.BehindBound + 1 + p.Compose() + o := len(orphaned) + if res = p.Compose(); len(res.Consumers.Behind) != 0 || len(orphaned) != o+1 || orphaned[o].State != occ.StateRaised || + !strings.Contains(orphaned[o].Explanation, "EVENTS.b_mod-c") || strings.Contains(orphaned[o].Explanation, "EVENTS.b_mod-c_tools") { + t.Fatalf("an orphaned durable: %+v, behind said %+v, orphaned said %+v", res.Consumers, behind[len(behind)-1], orphaned) + } + if p.Compose(); len(orphaned) != o+1 { + t.Fatalf("an orphan was said again: %+v", orphaned) + } + twin["last_delivered"] = "2026-10-23T21:00:00Z" + if res = p.Compose(); len(res.Consumers.Behind) != 1 || len(orphaned) != o+2 || orphaned[o+1].State != occ.StateCleared { + t.Fatalf("a durable whose twin delivered two days ago: %+v, orphaned said %+v", res.Consumers, orphaned) + } + made["pending"] = grants.BehindBound + p.Compose() }) // 774/AC13 (novox/hq ADR 0561 §7): after take-over a list in which more than one loaded user's hash changes, with no diff --git a/modules/mesh-grants/internal/grants/consumers.go b/modules/mesh-grants/internal/grants/consumers.go index dc141292d..88fc21371 100644 --- a/modules/mesh-grants/internal/grants/consumers.go +++ b/modules/mesh-grants/internal/grants/consumers.go @@ -7,6 +7,7 @@ import ( "slices" "sort" "strings" + "time" "github.com/nats-io/nats.go/jetstream" @@ -60,6 +61,36 @@ type heldConsumer struct { Filters []string `json:"filters"` MaxDeliver int `json:"max_deliver"` Pending int64 `json:"pending"` + // WaitingPulls and LastDelivered say whether it is read now: pulls waiting on it, and when it last delivered (H-A-472). + WaitingPulls int64 `json:"waiting_pulls"` + LastDelivered string `json:"last_delivered"` +} + +// orphanedWithin is how recent a twin's last delivery is for it to be current: within the day (design 47 H2, H-A-472). +const orphanedWithin = 24 * time.Hour + +// durableTwin names the twin of a module's durable: its program's _ and its tool runner's +// __tools (design 47 G, A-340), each the other's. +func durableTwin(name string) string { + if t, ok := strings.CutSuffix(name, "_tools"); ok { + return t + } + return name + "_tools" +} + +// orphaned says whether a held durable is an orphan (design 47 H2, H-A-472 (2)): no pulls waiting on it, while its twin +// on the same stream is current, with pulls waiting and a delivery within the day. A module that moved into the tool +// runner left its program's durable so, and one that runs its own program again its tool runner's. +func orphaned(h heldConsumer, held map[string]heldConsumer, now time.Time) bool { + if h.WaitingPulls > 0 { + return false + } + t, ok := held[durableTwin(h.Name)] + if !ok || t.WaitingPulls == 0 { + return false + } + last, err := time.Parse(time.RFC3339Nano, t.LastDelivered) + return err == nil && now.Sub(last) < orphanedWithin } // heldConsumers reads the consumers the bus holds on stream, by name. @@ -123,6 +154,7 @@ func (p *Part) consumers(res *Result, recs buscompose.Records, match string) { st.Wanted = len(wanted) seen := map[string]bool{} defer func() { p.behindSeen = seen }() + var orphans []string byStream := map[string][]buscompose.Consumer{} for _, mc := range wanted { byStream[mc.Consumer.Stream] = append(byStream[mc.Consumer.Stream], mc.Consumer) @@ -141,7 +173,13 @@ func (p *Part) consumers(res *Result, recs buscompose.Records, match string) { } if ok { if h.Pending > BehindBound { - if p.behindSeen[key] { + // An orphan is left out of the behind check and named in a finding of its own (H-A-472 (2)), so the + // alarm says a real backlog only. + switch { + case !p.behindSeen[key]: + case orphaned(h, held, p.now()): + orphans = append(orphans, key) + default: st.Behind = append(st.Behind, fmt.Sprintf("%s: %d pending", key, h.Pending)) } seen[key] = true @@ -216,6 +254,7 @@ func (p *Part) consumers(res *Result, recs buscompose.Records, match string) { } p.sayDiffer(st) p.sayBehind(st) + p.sayOrphaned(orphans) } // sayDiffer raises the warning condition mesh-grants.consumer-differs while an account consumer the part owns (from the @@ -347,6 +386,40 @@ func (p *Part) sayBehind(st *ConsumersState) { } } +// DurableOrphanedCondition is the key of the warning raised while an account consumer is an orphaned durable: far +// behind, no pulls waiting, its twin on the stream current (design 47 H2, H-A-472 (2)). +const DurableOrphanedCondition = "mesh-grants.durable-orphaned" + +// sayOrphaned raises the warning condition mesh-grants.durable-orphaned naming the orphaned durables the behind check +// left out, once per set, and clears it when none is left (they were removed, by nats_consumer_remove). +func (p *Part) sayOrphaned(orphans []string) { + if p.Say == nil { + return + } + why := strings.Join(orphans, ", ") + p.holdMu.Lock() + defer p.holdMu.Unlock() + if p.orphanedKnown && why == p.orphanedSaid { + return + } + c := oc.Condition{Key: DurableOrphanedCondition, Subject: oc.ConditionSubject{Scope: "module", ID: "mesh-grants"}} + if why == "" { + c.State = occ.StateCleared + } else { + c.Severity, c.State = occ.SeverityWarning, occ.StateRaised + c.Summary = fmt.Sprintf("%d durable consumers are orphans: nothing reads them, while their twin on the stream does", len(orphans)) + c.Headline = "A listener's old consumer is left over" + needsYou(&c, "their removal, with the bus module's nats_consumer_remove", "These durables are more than the bound behind "+ + "their stream's head with no pulls waiting, while their module's twin on the same stream (_ and "+ + "__tools) has pulls waiting and delivered within the day: a module that moved into or out of the tool "+ + "runner left them, and they are not said as messages piling up: "+why) + c.Resolved = "no orphaned durable is left" + } + if p.Say(c) == nil { + p.orphanedSaid, p.orphanedKnown = why, true + } +} + // sayWorker logs one named seat worker's outcome (created, held already) once per change, so the journal names each worker the // pass considers (A-413). func (p *Part) sayWorker(line string) { diff --git a/modules/mesh-grants/internal/grants/part.go b/modules/mesh-grants/internal/grants/part.go index 786065d85..99350c347 100644 --- a/modules/mesh-grants/internal/grants/part.go +++ b/modules/mesh-grants/internal/grants/part.go @@ -230,6 +230,9 @@ type Part struct { behindSaid string behindKnown bool behindSeen map[string]bool + // orphanedSaid is the orphaned durables last said as a condition, once orphanedKnown (consumers.go, H-A-472). + orphanedSaid string + orphanedKnown bool // fullCleared is whether the old history-full key was cleared since the start (full.go, ADR 0595 §6). fullCleared bool // filterRefused is, by consumer, the refusal of the last addition of filters, so a standing refusal is said once.