From 9005a2ee8ecbacbb29b8a0cd3130348ba7d9016f Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 14:03:55 +0100 Subject: [PATCH] Grants: the behind check leaves out an orphaned durable and names it apart About 110 durables left by modules that moved into or out of the tool runner pile up behind their stream's head and raised 'Messages pile up for a listener' for no real backlog. A durable with no pulls waiting whose twin on the stream (_ and __tools) has pulls waiting and delivered within the day is now left out of that check and named once in mesh-grants.durable-orphaned, cleared when gone. Builds: design 47 H2 H-A-472 (2), G A-340 --- modules/mesh-grants/grants_test.go | 30 +++++++- .../mesh-grants/internal/grants/consumers.go | 75 ++++++++++++++++++- modules/mesh-grants/internal/grants/part.go | 3 + 3 files changed, 106 insertions(+), 2 deletions(-) diff --git a/modules/mesh-grants/grants_test.go b/modules/mesh-grants/grants_test.go index 5ae14e1d8..975123922 100644 --- a/modules/mesh-grants/grants_test.go +++ b/modules/mesh-grants/grants_test.go @@ -4167,12 +4167,14 @@ func TestGrants(t *testing.T) { } // A held consumer that differs is never changed by a pass: it is raised as a condition naming it, and brought to // the composed configuration only by hand (grants_consumer_match), with no lease key too. - var said, behind []oc.Condition + var said, behind, orphaned []oc.Condition p.Say = func(c oc.Condition) error { switch c.Key { case grants.GrantMissingCondition, grants.RefusedCondition, grants.HeldModuleCondition, grants.FullCondition: // their own rows case grants.ConsumerBehindCondition: behind = append(behind, c) + case grants.DurableOrphanedCondition: + orphaned = append(orphaned, c) default: said = append(said, c) } @@ -4222,6 +4224,32 @@ func TestGrants(t *testing.T) { if len(res.Consumers.Behind) != 0 || behind[len(behind)-1].State != occ.StateCleared { t.Errorf("a consumer back within the bound: %+v, said %+v", res.Consumers, behind) } + // Design 47 H2, H-A-472 (2): a durable far behind with no pulls waiting, whose twin on the stream (b_mod-c_tools) has + // pulls waiting and delivered within the day, is an orphan: left out of the behind check, named once in + // mesh-grants.durable-orphaned, and cleared when it is gone; with its twin not current, it is behind again. + var twin map[string]any + for _, c := range b.consumers["EVENTS"] { + if c["name"] == "b_mod-c_tools" { + twin = c + } + } + twin["waiting_pulls"], twin["last_delivered"] = 1, "2026-10-25T21:00:00.5Z" + made["pending"] = grants.BehindBound + 1 + p.Compose() + o := len(orphaned) + if res = p.Compose(); len(res.Consumers.Behind) != 0 || len(orphaned) != o+1 || orphaned[o].State != occ.StateRaised || + !strings.Contains(orphaned[o].Explanation, "EVENTS.b_mod-c") || strings.Contains(orphaned[o].Explanation, "EVENTS.b_mod-c_tools") { + t.Fatalf("an orphaned durable: %+v, behind said %+v, orphaned said %+v", res.Consumers, behind[len(behind)-1], orphaned) + } + if p.Compose(); len(orphaned) != o+1 { + t.Fatalf("an orphan was said again: %+v", orphaned) + } + twin["last_delivered"] = "2026-10-23T21:00:00Z" + if res = p.Compose(); len(res.Consumers.Behind) != 1 || len(orphaned) != o+2 || orphaned[o+1].State != occ.StateCleared { + t.Fatalf("a durable whose twin delivered two days ago: %+v, orphaned said %+v", res.Consumers, orphaned) + } + made["pending"] = grants.BehindBound + p.Compose() }) // 774/AC13 (novox/hq ADR 0561 §7): after take-over a list in which more than one loaded user's hash changes, with no diff --git a/modules/mesh-grants/internal/grants/consumers.go b/modules/mesh-grants/internal/grants/consumers.go index dc141292d..88fc21371 100644 --- a/modules/mesh-grants/internal/grants/consumers.go +++ b/modules/mesh-grants/internal/grants/consumers.go @@ -7,6 +7,7 @@ import ( "slices" "sort" "strings" + "time" "github.com/nats-io/nats.go/jetstream" @@ -60,6 +61,36 @@ type heldConsumer struct { Filters []string `json:"filters"` MaxDeliver int `json:"max_deliver"` Pending int64 `json:"pending"` + // WaitingPulls and LastDelivered say whether it is read now: pulls waiting on it, and when it last delivered (H-A-472). + WaitingPulls int64 `json:"waiting_pulls"` + LastDelivered string `json:"last_delivered"` +} + +// orphanedWithin is how recent a twin's last delivery is for it to be current: within the day (design 47 H2, H-A-472). +const orphanedWithin = 24 * time.Hour + +// durableTwin names the twin of a module's durable: its program's _ and its tool runner's +// __tools (design 47 G, A-340), each the other's. +func durableTwin(name string) string { + if t, ok := strings.CutSuffix(name, "_tools"); ok { + return t + } + return name + "_tools" +} + +// orphaned says whether a held durable is an orphan (design 47 H2, H-A-472 (2)): no pulls waiting on it, while its twin +// on the same stream is current, with pulls waiting and a delivery within the day. A module that moved into the tool +// runner left its program's durable so, and one that runs its own program again its tool runner's. +func orphaned(h heldConsumer, held map[string]heldConsumer, now time.Time) bool { + if h.WaitingPulls > 0 { + return false + } + t, ok := held[durableTwin(h.Name)] + if !ok || t.WaitingPulls == 0 { + return false + } + last, err := time.Parse(time.RFC3339Nano, t.LastDelivered) + return err == nil && now.Sub(last) < orphanedWithin } // heldConsumers reads the consumers the bus holds on stream, by name. @@ -123,6 +154,7 @@ func (p *Part) consumers(res *Result, recs buscompose.Records, match string) { st.Wanted = len(wanted) seen := map[string]bool{} defer func() { p.behindSeen = seen }() + var orphans []string byStream := map[string][]buscompose.Consumer{} for _, mc := range wanted { byStream[mc.Consumer.Stream] = append(byStream[mc.Consumer.Stream], mc.Consumer) @@ -141,7 +173,13 @@ func (p *Part) consumers(res *Result, recs buscompose.Records, match string) { } if ok { if h.Pending > BehindBound { - if p.behindSeen[key] { + // An orphan is left out of the behind check and named in a finding of its own (H-A-472 (2)), so the + // alarm says a real backlog only. + switch { + case !p.behindSeen[key]: + case orphaned(h, held, p.now()): + orphans = append(orphans, key) + default: st.Behind = append(st.Behind, fmt.Sprintf("%s: %d pending", key, h.Pending)) } seen[key] = true @@ -216,6 +254,7 @@ func (p *Part) consumers(res *Result, recs buscompose.Records, match string) { } p.sayDiffer(st) p.sayBehind(st) + p.sayOrphaned(orphans) } // sayDiffer raises the warning condition mesh-grants.consumer-differs while an account consumer the part owns (from the @@ -347,6 +386,40 @@ func (p *Part) sayBehind(st *ConsumersState) { } } +// DurableOrphanedCondition is the key of the warning raised while an account consumer is an orphaned durable: far +// behind, no pulls waiting, its twin on the stream current (design 47 H2, H-A-472 (2)). +const DurableOrphanedCondition = "mesh-grants.durable-orphaned" + +// sayOrphaned raises the warning condition mesh-grants.durable-orphaned naming the orphaned durables the behind check +// left out, once per set, and clears it when none is left (they were removed, by nats_consumer_remove). +func (p *Part) sayOrphaned(orphans []string) { + if p.Say == nil { + return + } + why := strings.Join(orphans, ", ") + p.holdMu.Lock() + defer p.holdMu.Unlock() + if p.orphanedKnown && why == p.orphanedSaid { + return + } + c := oc.Condition{Key: DurableOrphanedCondition, Subject: oc.ConditionSubject{Scope: "module", ID: "mesh-grants"}} + if why == "" { + c.State = occ.StateCleared + } else { + c.Severity, c.State = occ.SeverityWarning, occ.StateRaised + c.Summary = fmt.Sprintf("%d durable consumers are orphans: nothing reads them, while their twin on the stream does", len(orphans)) + c.Headline = "A listener's old consumer is left over" + needsYou(&c, "their removal, with the bus module's nats_consumer_remove", "These durables are more than the bound behind "+ + "their stream's head with no pulls waiting, while their module's twin on the same stream (_ and "+ + "__tools) has pulls waiting and delivered within the day: a module that moved into or out of the tool "+ + "runner left them, and they are not said as messages piling up: "+why) + c.Resolved = "no orphaned durable is left" + } + if p.Say(c) == nil { + p.orphanedSaid, p.orphanedKnown = why, true + } +} + // sayWorker logs one named seat worker's outcome (created, held already) once per change, so the journal names each worker the // pass considers (A-413). func (p *Part) sayWorker(line string) { diff --git a/modules/mesh-grants/internal/grants/part.go b/modules/mesh-grants/internal/grants/part.go index 786065d85..99350c347 100644 --- a/modules/mesh-grants/internal/grants/part.go +++ b/modules/mesh-grants/internal/grants/part.go @@ -230,6 +230,9 @@ type Part struct { behindSaid string behindKnown bool behindSeen map[string]bool + // orphanedSaid is the orphaned durables last said as a condition, once orphanedKnown (consumers.go, H-A-472). + orphanedSaid string + orphanedKnown bool // fullCleared is whether the old history-full key was cleared since the start (full.go, ADR 0595 §6). fullCleared bool // filterRefused is, by consumer, the refusal of the last addition of filters, so a standing refusal is said once.