Merge commit 'c4521a2b' into feat/47-G-fix11-b2f
This commit is contained in:
@@ -0,0 +1,131 @@
|
||||
package process
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// ConsumersState is the name of the state a module keeps its durable consumers' facts in, one key per consumer by its name,
|
||||
// history 1 (design 47 G A-166, A-176): what a module's tool reads of a consumer, since a tool runs in a runtime of its own and
|
||||
// cannot read the consumer's information.
|
||||
const ConsumersState = "consumers"
|
||||
|
||||
// ConsumerFacts is one durable consumer as ConsumerLedger keeps it.
|
||||
type ConsumerFacts struct {
|
||||
// Stream and Name are the consumer's stream and its name.
|
||||
Stream string `json:"stream"`
|
||||
Name string `json:"name"`
|
||||
// Acknowledged is the stream sequence of the last message applied.
|
||||
Acknowledged uint64 `json:"acknowledged"`
|
||||
// Pending is the messages the consumer has not been given yet, as the last message said.
|
||||
Pending uint64 `json:"pending"`
|
||||
// LastEvent is the subject of the last event applied and LastAt its own `at`, or when the bus stored it where it has none.
|
||||
LastEvent string `json:"last-event,omitempty"`
|
||||
LastAt time.Time `json:"last-at,omitempty"`
|
||||
// OldestUnacknowledged is when the oldest message delivered and not yet applied was delivered; zero when none is.
|
||||
OldestUnacknowledged time.Time `json:"oldest-unacknowledged,omitempty"`
|
||||
}
|
||||
|
||||
// ConsumerSink stores one key of the module's state consumers.
|
||||
type ConsumerSink func(ctx context.Context, key string, value []byte) error
|
||||
|
||||
// ConsumerLedger writes a durable consumer's facts into the module's state consumers after each message delivered and each message
|
||||
// applied, so a tool reads them and nothing is read on a timer (design 47 G A-166, A-176). One ledger serves one consumer.
|
||||
type ConsumerLedger struct {
|
||||
stream, name string
|
||||
sink ConsumerSink
|
||||
now func() time.Time
|
||||
logf func(string, ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
facts ConsumerFacts
|
||||
unacks map[uint64]time.Time // delivered and not applied, by stream sequence
|
||||
}
|
||||
|
||||
// NewConsumerLedger is the ledger of the consumer name on stream, writing through sink; a nil now is the clock and a nil logf says
|
||||
// nothing.
|
||||
func NewConsumerLedger(stream, name string, sink ConsumerSink, now func() time.Time, logf func(string, ...any)) *ConsumerLedger {
|
||||
if now == nil {
|
||||
now = time.Now
|
||||
}
|
||||
if logf == nil {
|
||||
logf = func(string, ...any) {}
|
||||
}
|
||||
return &ConsumerLedger{stream: stream, name: name, sink: sink, now: now, logf: logf, unacks: map[uint64]time.Time{},
|
||||
facts: ConsumerFacts{Stream: stream, Name: name}}
|
||||
}
|
||||
|
||||
// Delivered notes that m was given to the consumer, and writes the facts; a message given again keeps its first delivery's time.
|
||||
func (l *ConsumerLedger) Delivered(m jetstream.Msg) {
|
||||
md, err := m.Metadata()
|
||||
if err != nil {
|
||||
l.logf("consumers: %s's message has no metadata, its delivery is not kept: %v", l.name, err)
|
||||
return
|
||||
}
|
||||
l.mu.Lock()
|
||||
// A message delivered again keeps the time it was first delivered, so one that never settles grows old (A-187): asked again
|
||||
// in this run, the time kept; after a restart, which kept none, the time the bus stored it, never later than its first
|
||||
// delivery.
|
||||
if _, known := l.unacks[md.Sequence.Stream]; !known {
|
||||
first := l.now().UTC()
|
||||
if md.NumDelivered > 1 && !md.Timestamp.IsZero() && md.Timestamp.Before(first) {
|
||||
first = md.Timestamp.UTC()
|
||||
}
|
||||
l.unacks[md.Sequence.Stream] = first
|
||||
}
|
||||
l.facts.Pending = md.NumPending
|
||||
l.mu.Unlock()
|
||||
l.write()
|
||||
}
|
||||
|
||||
// Applied notes that m was applied and acknowledged, at the event's own `at` (zero: when the bus stored it), and writes the facts.
|
||||
func (l *ConsumerLedger) Applied(m jetstream.Msg, at time.Time) {
|
||||
md, err := m.Metadata()
|
||||
if err != nil {
|
||||
l.logf("consumers: %s's message has no metadata, its application is not kept: %v", l.name, err)
|
||||
return
|
||||
}
|
||||
if at.IsZero() {
|
||||
at = md.Timestamp
|
||||
}
|
||||
l.mu.Lock()
|
||||
delete(l.unacks, md.Sequence.Stream)
|
||||
if md.Sequence.Stream > l.facts.Acknowledged {
|
||||
l.facts.Acknowledged = md.Sequence.Stream
|
||||
}
|
||||
l.facts.Pending = md.NumPending
|
||||
l.facts.LastEvent, l.facts.LastAt = m.Subject(), at.UTC()
|
||||
l.mu.Unlock()
|
||||
l.write()
|
||||
}
|
||||
|
||||
// Facts are the facts now.
|
||||
func (l *ConsumerLedger) Facts() ConsumerFacts {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
f := l.facts
|
||||
f.OldestUnacknowledged = time.Time{}
|
||||
for _, t := range l.unacks {
|
||||
if f.OldestUnacknowledged.IsZero() || t.Before(f.OldestUnacknowledged) {
|
||||
f.OldestUnacknowledged = t
|
||||
}
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
func (l *ConsumerLedger) write() {
|
||||
if l.sink == nil {
|
||||
return
|
||||
}
|
||||
data, err := json.Marshal(l.Facts())
|
||||
if err == nil {
|
||||
err = l.sink(context.Background(), l.name, data)
|
||||
}
|
||||
if err != nil {
|
||||
l.logf("consumers: the facts of %s are not written to the state %s: %v", l.name, ConsumersState, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package process
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
type ledgerMsg struct {
|
||||
jetstream.Msg
|
||||
subject string
|
||||
seq uint64
|
||||
pending uint64
|
||||
stored time.Time
|
||||
times uint64
|
||||
}
|
||||
|
||||
func (m ledgerMsg) Subject() string { return m.subject }
|
||||
func (m ledgerMsg) Metadata() (*jetstream.MsgMetadata, error) {
|
||||
return &jetstream.MsgMetadata{Sequence: jetstream.SequencePair{Stream: m.seq, Consumer: m.seq}, NumPending: m.pending, Timestamp: m.stored, NumDelivered: max(m.times, 1)}, nil
|
||||
}
|
||||
|
||||
func TestConsumerLedgerKeepsTheFactsAfterEachMessage(t *testing.T) {
|
||||
kept := map[string][]byte{}
|
||||
now := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC)
|
||||
l := NewConsumerLedger("EVENTS", "n1_m", func(_ context.Context, key string, value []byte) error { kept[key] = value; return nil }, func() time.Time { return now }, nil)
|
||||
read := func() ConsumerFacts {
|
||||
var f ConsumerFacts
|
||||
if err := json.Unmarshal(kept["n1_m"], &f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return f
|
||||
}
|
||||
a, b := ledgerMsg{subject: "mesh.seat.s.event.a", seq: 7, pending: 1}, ledgerMsg{subject: "mesh.seat.s.event.b", seq: 8, pending: 0}
|
||||
l.Delivered(a)
|
||||
now = now.Add(time.Minute)
|
||||
l.Delivered(b)
|
||||
if f := read(); f.Pending != 0 || !f.OldestUnacknowledged.Equal(now.Add(-time.Minute)) {
|
||||
t.Errorf("after two deliveries: %+v, want pending 0 and the first one's delivery as the oldest unacknowledged", f)
|
||||
}
|
||||
// A message delivered again keeps its first delivery's time (A-187): asked again in this run, and after a restart, where the
|
||||
// new ledger knows none, the time the bus stored it.
|
||||
now = now.Add(time.Minute)
|
||||
l.Delivered(ledgerMsg{subject: a.subject, seq: 7, pending: 0, times: 2})
|
||||
if f := read(); !f.OldestUnacknowledged.Equal(now.Add(-2 * time.Minute)) {
|
||||
t.Errorf("after a delivery again: %+v, want the first delivery kept as the oldest unacknowledged", f)
|
||||
}
|
||||
stored := now.Add(-time.Hour)
|
||||
again := NewConsumerLedger("EVENTS", "n1_m", func(_ context.Context, key string, value []byte) error { kept[key] = value; return nil }, func() time.Time { return now }, nil)
|
||||
again.Delivered(ledgerMsg{subject: a.subject, seq: 7, stored: stored, times: 3})
|
||||
if f := read(); !f.OldestUnacknowledged.Equal(stored) {
|
||||
t.Errorf("after a restart's delivery again: %+v, want the time the bus stored it", f)
|
||||
}
|
||||
now = now.Add(-time.Minute)
|
||||
at := now.Add(-time.Hour)
|
||||
l.Applied(a, at)
|
||||
if f := read(); f.Acknowledged != 7 || f.LastEvent != "mesh.seat.s.event.a" || !f.LastAt.Equal(at) || !f.OldestUnacknowledged.Equal(now) {
|
||||
t.Errorf("after the first applied: %+v, want sequence 7, its event and at, and the second one's delivery as the oldest", f)
|
||||
}
|
||||
l.Applied(b, time.Time{})
|
||||
if f := read(); f.Acknowledged != 8 || !f.OldestUnacknowledged.IsZero() {
|
||||
t.Errorf("after both applied: %+v, want sequence 8 and none unacknowledged", f)
|
||||
}
|
||||
}
|
||||
@@ -179,6 +179,32 @@ func (r *ProcessReader) Adopt(f *Follower, seat, event string, hear func(context
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// AdoptEvent hears a seat's event on the one subscription the followers adopted on that event share, with no picture and no
|
||||
// first view (design 47 G A-185, A-199): hear is handed each message of the event beside every adopted follower's handler,
|
||||
// and the message is acknowledged. The subscription is made at once when no follower made it, so a part that follows no
|
||||
// view of the seat still hears the event; adopted before Start, it is made at Start.
|
||||
func (r *ProcessReader) AdoptEvent(seat, event string, hear func(context.Context, []byte) error) error {
|
||||
if seat == "" || event == "" || hear == nil {
|
||||
return errors.New("views: an adopted event names its seat, its event and its handler")
|
||||
}
|
||||
subject := "mesh.seat." + seat + ".event." + event
|
||||
r.mu.Lock()
|
||||
if r.adopted == nil {
|
||||
r.adopted = map[string][]adoptedHear{}
|
||||
}
|
||||
fresh := len(r.adopted[subject]) == 0
|
||||
r.adopted[subject] = append(r.adopted[subject], adoptedHear{hear: hear})
|
||||
hub, handle := r.hub, r.handle
|
||||
r.mu.Unlock()
|
||||
if fresh && hub != nil {
|
||||
// one subscription per event, with every further part (design 47 G §532)
|
||||
if err := hub.Handle(subject+".>", handle); err != nil {
|
||||
r.logf("views: %s's event %s is not heard (%v)", r.Module, subject, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// adopt is Adopt's part under the lock; subject is the event's subject when no follower adopted it before.
|
||||
func (r *ProcessReader) adopt(f *Follower, seat, event string, hear func(context.Context, []byte) error) (*Picture, string, error) {
|
||||
r.mu.Lock()
|
||||
|
||||
@@ -800,6 +800,26 @@ func subViews(t *testing.T) {
|
||||
if len(a) != 1 || len(b) != 1 {
|
||||
t.Errorf("two adopted views on one process reader heard %v and %v", a, b)
|
||||
}
|
||||
// A-185, A-199: an event adopted with no picture is heard once on the same subscription, beside the views; a reader
|
||||
// following no view hears it too.
|
||||
var hooked []string
|
||||
hook := func(_ context.Context, d []byte) error { hooked = append(hooked, string(d)); return nil }
|
||||
if err := prr.AdoptEvent("s", "revision-said", hook); err != nil || prr.AdoptEvent("s", "", hook) == nil {
|
||||
t.Fatalf("AdoptEvent: %v, and one naming no event was taken", err)
|
||||
}
|
||||
prr.Hear(context.Background())(&seatMsg{subject: "mesh.seat.s.event.revision-said.main", data: []byte(`{"revision":8}`)})
|
||||
if len(a) != 2 || len(b) != 2 || len(hooked) != 1 || len(prr.Followers()) != 2 {
|
||||
t.Errorf("beside two views the adopted event heard %v (views %v, %v)", hooked, a, b)
|
||||
}
|
||||
alone := &ProcessReader{Module: "reader", Now: now}
|
||||
if err := alone.AdoptEvent("s", "revision-said", hook); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
am := &seatMsg{subject: "mesh.seat.s.event.revision-said.next", data: []byte(`{"revision":9}`)}
|
||||
alone.Hear(context.Background())(am)
|
||||
if len(hooked) != 2 || !am.acked {
|
||||
t.Errorf("a reader following no view heard %v, acked %v", hooked, am.acked)
|
||||
}
|
||||
pf.WaitFor(context.Background(), "4")
|
||||
ended <- true
|
||||
if fd := pfs.next(t); fd.Kind != ViewWaitsKind || fd.ID != "reader.s.revisions" {
|
||||
|
||||
Reference in New Issue
Block a user