Bound the read of bus-module events, keep S9 seeing when it fails, count the controller's own slow consumer once (hq issue 270, review of #326)
mesh/merge-gate head 45870ab31cc5 merged into trunk 3479a35462b2 — pass: builds build-agent, mesh-controller → ace, g14, novox, shanks; no bus step; every …
mesh/repo-check head 45870ab31cc5 merged into trunk 3479a35462b2 — pass: its merge-check.sh passed
mesh/delivery delivered
mesh/merge-gate head 45870ab31cc5 merged into trunk 3479a35462b2 — pass: builds build-agent, mesh-controller → ace, g14, novox, shanks; no bus step; every …
mesh/repo-check head 45870ab31cc5 merged into trunk 3479a35462b2 — pass: its merge-check.sh passed
mesh/delivery delivered
A storm of closures must not spend a watchdog tick, so the read stops at 2000 events and says the count is a floor. A failed read is its own condition rather than blinding S9, so the controller's own signals are still said. The controller's fall behind, told to its client and closed by the server, is one fault, counted once.
This commit is contained in:
@@ -154,7 +154,13 @@ func (s *Server) BusFaults(ctx context.Context, since time.Time) ([]Advisory, er
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
var closures []BusClosure
|
||||
var kinds []string
|
||||
read, cut := 0, false
|
||||
for {
|
||||
if read >= MaxBusEventsRead {
|
||||
cut = true
|
||||
break
|
||||
}
|
||||
read++
|
||||
wait, cancel := context.WithTimeout(ctx, readQuiet)
|
||||
msg, err := sub.NextMsgWithContext(wait)
|
||||
cancel()
|
||||
@@ -174,11 +180,23 @@ func (s *Server) BusFaults(ctx context.Context, since time.Time) ([]Advisory, er
|
||||
break
|
||||
}
|
||||
}
|
||||
return BusClosures(closures, kinds, since), nil
|
||||
out := BusClosures(closures, kinds, since)
|
||||
if cut {
|
||||
// Said, never silent: a busy hour is read up to the bound, and each advisory says it is a floor.
|
||||
for i := range out {
|
||||
out[i].Said += fmt.Sprintf(" (at least: the read stopped at %d of the bus module's events this hour)",
|
||||
MaxBusEventsRead)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// MaxBusEventsRead bounds one tick's read of the bus module's events: a watchdog tick must not spend itself
|
||||
// reading a storm of closures back. A read cut there says so in every advisory it makes.
|
||||
var MaxBusEventsRead = 2000
|
||||
|
||||
// MergeAdvisories is two lists of advisories as one: the same kind about the same thing is one advisory,
|
||||
// counting both, said by the newer. The controller's own slow consumer as its client library sees it and
|
||||
// counted once (the larger count of the two), said by the newer. The controller's own slow consumer as its client library sees it and
|
||||
// as the server closed it are the same user's (`controller`).
|
||||
func MergeAdvisories(a, b []Advisory) []Advisory {
|
||||
out := append([]Advisory(nil), a...)
|
||||
@@ -195,7 +213,11 @@ func MergeAdvisories(a, b []Advisory) []Advisory {
|
||||
continue
|
||||
}
|
||||
x := &out[i]
|
||||
x.Count += y.Count
|
||||
// The same fault seen twice, not two faults: the controller's own slow consumer is told to its client
|
||||
// library and closed by the server, and is counted once — by whichever saw more of it.
|
||||
if y.Count > x.Count {
|
||||
x.Count = y.Count
|
||||
}
|
||||
if y.First.Before(x.First) {
|
||||
x.First = y.First
|
||||
}
|
||||
|
||||
@@ -94,7 +94,28 @@ func TestTheControllersOwnSlowConsumerAndItsClosedConnectionAreOneAdvisory(t *te
|
||||
closed := BusClosures([]BusClosure{{User: "controller", CID: 5, Reason: "Slow Consumer (Pending Bytes)",
|
||||
Opened: at.Add(-time.Hour), Closed: at.Add(time.Minute)}}, []string{AdvisorySlowConsumer}, at.Add(-time.Hour))
|
||||
got := MergeAdvisories(own, closed)
|
||||
if len(got) != 1 || got[0].Count != 2 || !got[0].Last.Equal(at.Add(time.Minute)) || got[0].User != "controller" {
|
||||
t.Fatalf("merged as %+v", got)
|
||||
if len(got) != 1 || got[0].Count != 1 || !got[0].Last.Equal(at.Add(time.Minute)) || got[0].User != "controller" {
|
||||
t.Fatalf("merged as %+v: one fall behind, told to the client and closed by the server, is counted once", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **A busy hour is read up to a bound, and says it was cut** (review of #326): a storm of closures must not
|
||||
// spend a watchdog tick, and a count cut short is a floor, said as one.
|
||||
func TestNatsBusFaultsReadAtMostTheBoundAndSaySo(t *testing.T) {
|
||||
s := aBusWithEvents(t)
|
||||
was := MaxBusEventsRead
|
||||
MaxBusEventsRead = 2
|
||||
t.Cleanup(func() { MaxBusEventsRead = was })
|
||||
now := time.Now().UTC()
|
||||
for i := range 3 {
|
||||
sayClosure(t, s, BusSlowConsumerEvent, BusClosure{User: "laptop.claude-code", CID: uint64(10 + i),
|
||||
Reason: "Slow Consumer (Pending Bytes)", Opened: now.Add(-time.Hour / 2), Closed: now.Add(-time.Duration(3-i) * time.Minute)})
|
||||
}
|
||||
got, err := s.BusFaults(t.Context(), now.Add(-time.Hour))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Count != 2 || !strings.Contains(got[0].Said, "at least: the read stopped at 2") {
|
||||
t.Fatalf("a read past its bound read as %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user