The views test read NumAckPending as soon as the picture and the adopted handler had said the last event, but the reader acknowledges after both, and the bus takes acks on its own; on the loaded build node (pipeline main.54) one ack was still pending. The reader binds the module's durable once (the hub's one pull); the guard of H-A-196 is not involved. The test now publishes a mark the hub hands after every event before it, acknowledged with the bus's answer, and reads that the ack floor reached the mark; an event the owner says after it may still wait. Builds: design 47 H §774, H-A-196, H-A-197
329 lines
13 KiB
Go
329 lines
13 KiB
Go
//go:build integration
|
|
|
|
package views_test
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/nats-io/nats.go"
|
|
"github.com/nats-io/nats.go/jetstream"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/process"
|
|
"git.novox.be/novox/mesh-sdk/go/testbus"
|
|
"git.novox.be/novox/mesh-sdk/go/views"
|
|
"git.novox.be/novox/mesh-sdk/go/views/viewstest"
|
|
)
|
|
|
|
// An owner's change is an ordinary event on the shared events stream (novox/hq ADR 0635 §5): a reader that follows the
|
|
// owner's state by its read tool hears it on the module's durable consumer of that stream, with no stream of the owner's,
|
|
// and a message that is no views event goes to Other.
|
|
func subAReaderHearsAnOwnersChangesAsOrdinaryEvents(t *testing.T) {
|
|
nc, err := nats.Connect(testbus.URL(t))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer nc.Close()
|
|
js, err := jetstream.New(nc)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
cred := process.Account{Node: "n1", Module: "reader"}
|
|
kv := viewstest.Serve(t, ctx, js, nc, "o", "modules", "o.o_read_modules", cred.EventConsumerName())
|
|
if _, err := kv.Put(ctx, "a", []byte(`{"n":1}`)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
other := make(chan string, 1)
|
|
r := &views.ProcessReader{Module: "reader", Other: func(m jetstream.Msg) { other <- m.Subject(); _ = m.Ack() }, Logf: t.Logf,
|
|
Call: func(tool string, body any) (json.RawMessage, error) {
|
|
raw, _ := json.Marshal(body)
|
|
m, err := nc.Request("mesh.mod.o.call.o_read_modules.n0", raw, 5*time.Second)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return m.Data, nil
|
|
}}
|
|
if _, err := r.Follow("o", "modules", "o.o_read_modules"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
statusKV, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "reader_first-views", Storage: jetstream.MemoryStorage})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.Start(ctx, js, cred, process.AwaitBucket(ctx, js, nil, "reader_first-views", "", t.Logf))
|
|
<-r.Ready()
|
|
pic, err := r.KeyValue(ctx, "o_modules")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
w, err := pic.Watch(ctx, "b")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := kv.Put(ctx, "b", []byte(`{"n":2}`)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for e := range w.Updates() {
|
|
if e != nil && e.Key() == "b" {
|
|
break
|
|
}
|
|
}
|
|
if s := r.Followers()[0].Status(); s.Asks != 1 || s.Keys != 2 {
|
|
t.Fatalf("the change was not heard as an event alone: %+v", s)
|
|
}
|
|
if _, err := js.Publish(ctx, "mesh.mod.o.event.something-else", []byte("{}")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := <-other; got != "mesh.mod.o.event.something-else" {
|
|
t.Fatalf("other got %s", got)
|
|
}
|
|
// Across bursts that write the same keys over and over, the reader's copy stays its owner's, by the events alone: a
|
|
// revision written over before the owner's watch was handed it is no gap, and the owner's mark after each burst asks
|
|
// nothing of a reader that holds it (issue 1966).
|
|
all, err := pic.WatchAll(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var last uint64
|
|
for round := range 300 {
|
|
for range 3 {
|
|
if last, err = kv.Put(ctx, fmt.Sprintf("k%d", round%5), fmt.Appendf(nil, `{"round":%d}`, round)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
}
|
|
f := r.Followers()[0]
|
|
wait, stop := context.WithTimeout(ctx, 20*time.Second)
|
|
defer stop()
|
|
for f.Status().Revision < last {
|
|
select {
|
|
case <-all.Updates():
|
|
case <-wait.Done():
|
|
t.Fatalf("the reader stayed behind its owner at %d: %+v", last, f.Status())
|
|
}
|
|
}
|
|
for i := range 5 {
|
|
e, err := kv.Get(ctx, fmt.Sprintf("k%d", i))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if v, _ := f.Get(e.Key()); string(v) != string(e.Value()) {
|
|
t.Errorf("%s: the reader holds %s, its owner %s", e.Key(), v, e.Value())
|
|
}
|
|
}
|
|
if s := f.Status(); s.Asks != 1 {
|
|
t.Errorf("the reader asked its first view %d times across the bursts (%s): a revision written over was taken as a gap", s.Asks, s.Why)
|
|
}
|
|
// G §532, §614: a dropped view's status leaves the state first-views, and so its row of <module>_first_views.
|
|
if _, err := statusKV.Get(ctx, views.StatusKey("o", "modules")); err != nil {
|
|
t.Fatalf("the status was not kept: %v", err)
|
|
}
|
|
if err := r.Drop("o", "modules"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := statusKV.Get(ctx, views.StatusKey("o", "modules")); !errors.Is(err, jetstream.ErrKeyNotFound) {
|
|
t.Errorf("the dropped view's status is still kept: %v", err)
|
|
}
|
|
}
|
|
|
|
// A module has one durable consumer on the shared events stream, so the loops that hear events in one module must share
|
|
// one pull (process.EventHub): two handlers and the views reader on the hub each get their own messages and no other's,
|
|
// and a message nobody takes is acknowledged.
|
|
func subTheViewsReaderAndTwoHandlersShareTheModulesOneDurable(t *testing.T) {
|
|
nc, err := nats.Connect(testbus.URL(t))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer nc.Close()
|
|
js, err := jetstream.New(nc)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
cred := process.Account{Node: "n1", Module: "reader"}
|
|
kv := viewstest.Serve(t, ctx, js, nc, "o", "modules", "o.o_read_modules", cred.EventConsumerName())
|
|
hub := process.NewEventHub(t.Logf)
|
|
got := map[string]chan string{"a": make(chan string, 8), "b": make(chan string, 8)}
|
|
for name, pattern := range map[string]string{"a": "mesh.mod.p.event.a", "b": "mesh.mod.q.event.b.*"} {
|
|
// The handlers' acknowledgement waits for the bus's answer, so a message a handler handed on is acknowledged
|
|
// on the bus, and every one the hub handed before it too (acks of one consumer are taken in order).
|
|
if err := hub.Handle(pattern, func(m jetstream.Msg) { _ = m.DoubleAck(ctx); got[name] <- m.Subject() }); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
found := &seenFindings{put: make(chan map[string]any, 8)}
|
|
r := &views.ProcessReader{Module: "reader", Hub: hub, Logf: t.Logf, Findings: func() views.WireWriter { return found },
|
|
Call: func(tool string, body any) (json.RawMessage, error) {
|
|
raw, _ := json.Marshal(body)
|
|
m, err := nc.Request("mesh.mod.o.call.o_read_modules.n0", raw, 5*time.Second)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return m.Data, nil
|
|
}}
|
|
if _, err := r.Follow("o", "modules", "o.o_read_modules"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// H-A-166: a seat's event said on its bare subject (as the seat mesh-catalog says registration-changed) and one with
|
|
// further parts both reach the one adopted handler, each once.
|
|
if info, err := js.Stream(ctx, process.EventsStream); err != nil {
|
|
t.Fatal(err)
|
|
} else if cfg := info.CachedInfo().Config; !slices.Contains(cfg.Subjects, "mesh.seat.*.event.>") {
|
|
cfg.Subjects = append(cfg.Subjects, "mesh.seat.*.event.>")
|
|
if _, err := js.UpdateStream(ctx, cfg); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
seatHeard := make(chan string, 8)
|
|
if err := r.AdoptEvent("s", "registration-changed", func(_ context.Context, d []byte) error { seatHeard <- string(d); return nil }); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.Start(ctx, js, cred, nil)
|
|
if err := hub.Start(ctx, js, cred); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
<-r.Ready()
|
|
pic, err := r.KeyValue(ctx, "o_modules")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
w, err := pic.Watch(ctx, "b")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, s := range []string{"mesh.mod.p.event.a", "mesh.mod.q.event.b.c", "mesh.mod.z.event.nobody", "mesh.mod.p.event.a",
|
|
"mesh.seat.s.event.registration-changed", "mesh.seat.s.event.registration-changed.m"} {
|
|
if _, err := js.Publish(ctx, s, []byte(`"`+s+`"`)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if _, err := kv.Put(ctx, "b", []byte(`{"n":2}`)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for e := range w.Updates() {
|
|
if e != nil && e.Key() == "b" {
|
|
break
|
|
}
|
|
}
|
|
for _, want := range []struct{ who, subject string }{{"a", "mesh.mod.p.event.a"}, {"b", "mesh.mod.q.event.b.c"}, {"a", "mesh.mod.p.event.a"}} {
|
|
if s := <-got[want.who]; s != want.subject {
|
|
t.Fatalf("handler %s got %s, wanted %s", want.who, s, want.subject)
|
|
}
|
|
}
|
|
for _, want := range []string{`"mesh.seat.s.event.registration-changed"`, `"mesh.seat.s.event.registration-changed.m"`} {
|
|
select {
|
|
case d := <-seatHeard:
|
|
if d != want {
|
|
t.Fatalf("the adopted seat event heard %s, wanted %s", d, want)
|
|
}
|
|
case <-time.After(10 * time.Second):
|
|
t.Fatalf("the adopted seat event never heard %s", want)
|
|
}
|
|
}
|
|
if len(got["a"])+len(got["b"])+len(seatHeard) != 0 {
|
|
t.Fatal("a handler heard a message that was another's, or one twice")
|
|
}
|
|
// The reader acknowledges an event after it took it into its picture and the adopted handler returned, so the
|
|
// picture and the handler say nothing of its ack: one more event, handed after them in order and acknowledged with
|
|
// the bus's answer, is the mark that every ack before it has landed (the build node, main.54, read one pending). An
|
|
// event said after the mark (the owner's own mark of its burst) may still wait for its ack, so the floor is read.
|
|
mark, err := js.Publish(ctx, "mesh.mod.p.event.a", []byte(`"the mark"`))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
select {
|
|
case <-got["a"]:
|
|
case <-time.After(10 * time.Second):
|
|
t.Fatal("the mark after the handled events was not heard")
|
|
}
|
|
cons, err := js.Consumer(ctx, process.EventsStream, cred.EventConsumerName())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if info, err := cons.Info(ctx); err != nil || info.AckFloor.Stream < mark.Sequence {
|
|
t.Fatalf("a message up to the mark %d was left unacknowledged: %+v %v", mark.Sequence, info, err)
|
|
}
|
|
// H-A-169: the one event no handler takes is the module's finding event-unhandled, counted once, naming its subject,
|
|
// though the reader started twice.
|
|
r.Start(ctx, js, cred, nil)
|
|
select {
|
|
case f := <-found.put:
|
|
summary, _ := f["summary"].(string)
|
|
if f["kind"] != views.EventUnhandledKind || f["id"] != "reader" || !strings.Contains(summary, "took 1 events") ||
|
|
!strings.Contains(summary, "mesh.mod.z.event.nobody") {
|
|
t.Fatalf("the unhandled event's finding: %v", f)
|
|
}
|
|
case <-time.After(10 * time.Second):
|
|
t.Fatal("the unhandled event wrote no finding")
|
|
}
|
|
if _, err := js.Publish(ctx, "mesh.mod.z.event.nobody", []byte("{}")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := js.Publish(ctx, "mesh.mod.p.event.a", []byte("{}")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
select {
|
|
case <-got["a"]: // the hub hands messages in order: every hook of the event before has run
|
|
case <-time.After(10 * time.Second):
|
|
t.Fatal("the handled event after the unhandled one was not heard")
|
|
}
|
|
select {
|
|
case f := <-found.put:
|
|
if summary, _ := f["summary"].(string); !strings.Contains(summary, "took 2 events") || len(found.put) != 0 {
|
|
t.Fatalf("after a second start one event is counted once: %v, and %d more", f, len(found.put))
|
|
}
|
|
default:
|
|
t.Fatal("the unhandled event after the second start wrote no finding")
|
|
}
|
|
// A hub started before the bus holds its durable says so, and binds at a later start: a module is not lost for the
|
|
// order in which the bus makes its consumer.
|
|
late := process.Account{Node: "n1", Module: "late"}
|
|
lateHub := process.NewEventHub(t.Logf)
|
|
heard := make(chan string, 1)
|
|
if err := lateHub.Handle("mesh.mod.l.event.>", func(m jetstream.Msg) { _ = m.Ack(); heard <- m.Subject() }); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer lateHub.Stop()
|
|
if err := lateHub.Start(ctx, js, late); err == nil {
|
|
t.Fatal("a hub with no durable on the bus started")
|
|
}
|
|
if _, err := js.CreateOrUpdateConsumer(ctx, process.EventsStream, jetstream.ConsumerConfig{Durable: late.EventConsumerName(),
|
|
AckPolicy: jetstream.AckExplicitPolicy, FilterSubjects: []string{"mesh.mod.l.event.>"}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := lateHub.Start(ctx, js, late); err != nil {
|
|
t.Fatalf("the hub did not bind at its second start: %v", err)
|
|
}
|
|
if _, err := js.Publish(ctx, "mesh.mod.l.event.x", []byte("{}")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
select {
|
|
case <-heard:
|
|
case <-time.After(10 * time.Second):
|
|
t.Fatal("the hub started late heard nothing")
|
|
}
|
|
}
|
|
|
|
// TestAReaderHearsAnOwnersChangesAsOrdinaryEvents runs the tests of this package that were merged to keep the module within its test budget (design 47 G §419), each as a subtest.
|
|
func TestAReaderHearsAnOwnersChangesAsOrdinaryEvents(t *testing.T) {
|
|
t.Run("AReaderHearsAnOwnersChangesAsOrdinaryEvents", subAReaderHearsAnOwnersChangesAsOrdinaryEvents)
|
|
t.Run("TheViewsReaderAndTwoHandlersShareTheModulesOneDurable", subTheViewsReaderAndTwoHandlersShareTheModulesOneDurable)
|
|
t.Run("WriterSaysItsEventsAndHearsAsksOnTheBus", subWriterSaysItsEventsAndHearsAsksOnTheBus)
|
|
}
|
|
|
|
// seenFindings is a findings writer that hands each finding written to the test.
|
|
type seenFindings struct{ put chan map[string]any }
|
|
|
|
func (f *seenFindings) WriteWire(_ context.Context, w map[string]any) error { f.put <- w; return nil }
|
|
func (f *seenFindings) Clear(context.Context, string, string, string, string) error { return nil }
|
|
func (f *seenFindings) ModuleNode() (string, string) { return "reader", "n1" }
|