jschoubben
fe9533c186
Live fault 2026-11-08: node-engines on two machines logged only "nats: slow consumer, messages dropped" and applied nothing. The client's watch sends into its channel from inside the subscription's callback; KVWatch forwarded that channel to its reader blocking, and the reader (composing for minutes) stopped draining, so the subscription's pending limit overflowed. Quick fix: a larger buffer on the watch's channel; it only moves the limit. Thorough fix (chosen): the forwarder always drains the client's channel and, while its reader is busy, coalesces the pending updates by key (newest value and revision, first-seen order, Caught kept in its place), so no reader's pace can back the client up, whatever the burst. A slow consumer error now names its subscription and what it holds pending. Claude-Session: https://claude.ai/code/session_01NcbgfFfQ4Z9UE5jK32PmE5
69 lines
2.1 KiB
Go
69 lines
2.1 KiB
Go
package link
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestForwardCoalescing(t *testing.T) {
|
|
u := func(k string, rev uint64) KVUpdate { return KVUpdate{Key: k, Value: []byte{byte(rev)}, Revision: rev} }
|
|
caught := KVUpdate{Caught: true}
|
|
tests := []struct {
|
|
name string
|
|
in []KVUpdate
|
|
want []KVUpdate
|
|
}{
|
|
{"newest per key, first-seen order", []KVUpdate{u("a", 1), u("b", 2), u("a", 3), u("b", 4), u("a", 5)}, []KVUpdate{u("a", 5), u("b", 4)}},
|
|
{"caught keeps its place", []KVUpdate{u("a", 1), u("b", 2), caught, u("c", 3), u("a", 4)}, []KVUpdate{u("a", 4), u("b", 2), caught, u("c", 3)}},
|
|
{"delete is the newest", []KVUpdate{u("a", 1), {Key: "a", Revision: 2, Deleted: true}}, []KVUpdate{{Key: "a", Revision: 2, Deleted: true}}},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
in := make(chan KVUpdate) // unbuffered: every send must be taken at once, as the client's callback needs
|
|
out := make(chan KVUpdate) // unbuffered and not read yet
|
|
done := make(chan struct{})
|
|
go func() { forwardCoalescing(ctx, in, func(x KVUpdate) KVUpdate { return x }, out); close(done) }()
|
|
for _, x := range tt.in {
|
|
select {
|
|
case in <- x:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("the producer was held back by a reader that does not read")
|
|
}
|
|
}
|
|
close(in)
|
|
var got []KVUpdate
|
|
for x := range out2(out, done) {
|
|
got = append(got, x)
|
|
}
|
|
if len(got) != len(tt.want) {
|
|
t.Fatalf("got %d updates %v, want %d %v", len(got), got, len(tt.want), tt.want)
|
|
}
|
|
for i := range got {
|
|
if got[i].Key != tt.want[i].Key || got[i].Revision != tt.want[i].Revision || got[i].Caught != tt.want[i].Caught || got[i].Deleted != tt.want[i].Deleted {
|
|
t.Fatalf("update %d is %+v, want %+v", i, got[i], tt.want[i])
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// out2 reads out until the forwarder has ended.
|
|
func out2(out <-chan KVUpdate, done <-chan struct{}) <-chan KVUpdate {
|
|
r := make(chan KVUpdate)
|
|
go func() {
|
|
defer close(r)
|
|
for {
|
|
select {
|
|
case x := <-out:
|
|
r <- x
|
|
case <-done:
|
|
return
|
|
}
|
|
}
|
|
}()
|
|
return r
|
|
}
|