diff --git a/go/process/event_hub.go b/go/process/event_hub.go index 0af4a850..bd6a0532 100644 --- a/go/process/event_hub.go +++ b/go/process/event_hub.go @@ -26,6 +26,8 @@ type EventHub struct { routes []hubRoute cons jetstream.ConsumeContext bound []func(created time.Time) + // unhandled are told each subject no handler took (OnUnhandled). + unhandled []func(subject string) // kept are the module's durables on the kept seat events' streams it handles (StartKept). kept []jetstream.ConsumeContext } @@ -72,6 +74,15 @@ func (h *EventHub) OnBound(f func(created time.Time)) { h.mu.Unlock() } +// OnUnhandled registers f, told the subject of each message no handler takes, after the hub said it in the log and +// acknowledged it: the module's process reader counts them in its finding `event-unhandled` (design 47 H, H-A-166, +// H-A-169). +func (h *EventHub) OnUnhandled(f func(subject string)) { + h.mu.Lock() + h.unhandled = append(h.unhandled, f) + h.mu.Unlock() +} + // Patterns are the registered subject patterns, in order. func (h *EventHub) Patterns() []string { h.mu.RLock() @@ -235,10 +246,14 @@ func (h *EventHub) hear(m jetstream.Msg) { break } } + told := h.unhandled h.mu.RUnlock() if take == nil { h.logf("events: %s is taken by no handler of this module, acknowledged", m.Subject()) _ = m.Ack() + for _, f := range told { + f(m.Subject()) + } return } take(m) diff --git a/go/process/event_hub_test.go b/go/process/event_hub_test.go index 16797254..c6ac390d 100644 --- a/go/process/event_hub_test.go +++ b/go/process/event_hub_test.go @@ -37,6 +37,15 @@ func TestTheHubMatchesSubjectsAndRefusesOverlappingPatterns(t *testing.T) { if err := h.Handle("mesh.mod.q.event.>", nop); err != nil { t.Fatal(err) } + // H-A-169: a message no handler takes is acknowledged and told to each OnUnhandled hook, with its subject; one taken is not. + var told []string + h.OnUnhandled(func(subject string) { told = append(told, subject) }) + taken, lost := &bindMsg{subject: "mesh.mod.q.event.x"}, &bindMsg{subject: "mesh.seat.s.event.nobody"} + h.hear(taken) + h.hear(lost) + if !lost.acked || len(told) != 1 || told[0] != "mesh.seat.s.event.nobody" { + t.Errorf("the unhandled message: acked %v, told %v", lost.acked, told) + } // W1.12: a seat's event kept in a stream of its own is heard on one bus name and one durable, the maker's and the // hearer's alike; an event kept last per subject is delivered from the last of each subject. The name is the // contract's Stream as it stands (seatdef.KeptStream), never made again here: a dotted or unprefixed name is diff --git a/go/views/findings.go b/go/views/findings.go index 9d02aa51..42326a3a 100644 --- a/go/views/findings.go +++ b/go/views/findings.go @@ -23,6 +23,9 @@ const ( // EntryNotFoundKind is a part's finding of an entry by fingerprint the seat mesh-catalog's `entry` answered it does not hold // (design 47 G §614, A-15, A-103): scope the part, id the fingerprint. EntryNotFoundKind = "entry-not-found" + // EventUnhandledKind is a process's finding of the events its durable consumer delivered that no handler of the module + // took, one per module, counted since the process started (design 47 H, H-A-166, H-A-169). + EventUnhandledKind = "event-unhandled" ) // viewFinding is what a views part says is wrong with a state it owns or follows, in plain fields: kind and id name it, summary @@ -120,3 +123,11 @@ func WriteEntryNotFound(w WireWriter, fingerprint string, since, at time.Time) e func ClearEntryNotFound(w WireWriter, fingerprint string) error { return viewFinding{kind: EntryNotFoundKind, id: fingerprint}.drop(w) } + +// unhandledFinding is the module's one finding of the events no handler took since its process started (H-A-169): its id the +// module, its summary the count and the last subject. +func unhandledFinding(module string, count int, last string, since, at time.Time) viewFinding { + summary := fmt.Sprintf("%s took %d events by no handler since its process started at %s, the last on %s: what the module consumes it does not act on", + module, count, since.UTC().Format(time.RFC3339), last) + return viewFinding{kind: EventUnhandledKind, id: module, summary: summary, headline: fmt.Sprintf("%s hears events it does not handle", module), at: at.UTC()} +} diff --git a/go/views/process_reader.go b/go/views/process_reader.go index 698f07e3..df862b12 100644 --- a/go/views/process_reader.go +++ b/go/views/process_reader.go @@ -73,6 +73,12 @@ type ProcessReader struct { asked bool begun map[string]context.CancelFunc status *process.AwaitedBucket + // unheard counts the events no handler of the module took since Start, the last of them on lastUnheard (H-A-169). + unheard int + lastUnheard string + startedAt time.Time + // hooked is the hub the count of unhandled events is wired on, so a second Start counts no event twice. + hooked *process.EventHub } // adoptedHear is one adopted follower's handler of its seat event, by its picture's bucket. @@ -175,8 +181,7 @@ func (r *ProcessReader) Adopt(f *Follower, seat, event string, hear func(context hub, handle := r.hub, r.handle r.mu.Unlock() if subject != "" && hub != nil { - // one subscription per event, with every further part (design 47 G §532) - if err := hub.Handle(subject+".>", handle); err != nil { + if err := handleEvent(hub, subject, handle); err != nil { r.logf("views: %s's event %s is not heard (%v): the view %s is its first view's alone, asked again by hand", r.Module, subject, err, p.bucket) } } @@ -202,8 +207,7 @@ func (r *ProcessReader) AdoptEvent(seat, event string, hear func(context.Context hub, handle := r.hub, r.handle r.mu.Unlock() if fresh && hub != nil { - // one subscription per event, with every further part (design 47 G §532) - if err := hub.Handle(subject+".>", handle); err != nil { + if err := handleEvent(hub, subject, handle); err != nil { r.logf("views: %s's event %s is not heard (%v)", r.Module, subject, err) } } @@ -397,6 +401,50 @@ func (r *ProcessReader) Hear(ctx context.Context) func(jetstream.Msg) { } r.logf("views: %s is no views event of an owner followed, acknowledged", m.Subject()) _ = m.Ack() + r.unhandled(m.Subject()) + } +} + +// now is the reader's clock. +func (r *ProcessReader) now() time.Time { + if r.Now != nil { + return r.Now() + } + return time.Now() +} + +// unhandled counts an event no handler of the module took, said in the log already, in the module's finding +// `event-unhandled` (design 47 H, H-A-166, H-A-169); with no findings writer the log line stands alone. +func (r *ProcessReader) unhandled(subject string) { + if r.Findings == nil { + return + } + r.mu.Lock() + r.unheard++ + r.lastUnheard = subject + count, since := r.unheard, r.startedAt + r.mu.Unlock() + w := r.Findings() + if w == nil { + return + } + if err := unhandledFinding(r.Module, count, subject, since, r.now()).put(w); err != nil { + r.logf("views: the finding %s of %s is not written: %v", EventUnhandledKind, r.Module, err) + } +} + +// clearUnhandled clears the module's finding `event-unhandled` at the process's start: a new build may handle the event +// (H-A-169). +func (r *ProcessReader) clearUnhandled() { + if r.Findings == nil { + return + } + w := r.Findings() + if w == nil { + return + } + if err := (viewFinding{kind: EventUnhandledKind, id: r.Module}).drop(w); err != nil { + r.logf("views: the finding %s of %s is not cleared at its start: %v", EventUnhandledKind, r.Module, err) } } @@ -849,7 +897,21 @@ func (r *ProcessReader) Start(ctx context.Context, js jetstream.JetStream, cred }) hear := r.Hear(ctx) err := hub.Handle(pattern, hear) + if r.Findings != nil { + // H-A-169: the events no handler of the module takes are its finding, cleared at each start + r.clearUnhandled() + r.mu.Lock() + wire := r.hooked != hub + r.hooked = hub + r.mu.Unlock() + if wire { + hub.OnUnhandled(r.unhandled) + } + } r.mu.Lock() + if r.startedAt.IsZero() { + r.startedAt = r.now() // the process's start; a second Start counts on from it + } r.status = status if !own { // an event adopted after the start is handled on the module's hub (Adopt); the reader's own hub takes every one @@ -861,9 +923,8 @@ func (r *ProcessReader) Start(ctx context.Context, js jetstream.JetStream, cred subjects := slices.Sorted(maps.Keys(r.adopted)) r.mu.Unlock() for _, subject := range subjects { - // one subscription per event, with every further part (design 47 G §532) if err == nil { - err = hub.Handle(subject+".>", hear) + err = handleEvent(hub, subject, hear) } } } @@ -891,6 +952,16 @@ func (r *ProcessReader) Start(ctx context.Context, js jetstream.JetStream, cred r.Ask(ctx, said) } +// handleEvent registers handle on the hub for a seat's event on its bare subject and on every further part after it +// (`.>`): one handler, both forms, as the seat's contract says the event on either (design 47 H, H-A-166; one +// subscription per event, design 47 G §532). The two patterns do not overlap, so a message reaches the handler once. +func handleEvent(hub *process.EventHub, subject string, handle func(jetstream.Msg)) error { + if err := hub.Handle(subject, handle); err != nil { + return err + } + return hub.Handle(subject+".>", handle) +} + // adoptedHears are the handlers of every follower adopted on the event whose subject is subject or has further parts // after it (`...`, design 47 G §521, §532), or nil. func adoptedHears(adopted map[string][]adoptedHear, subject string) []adoptedHear { diff --git a/go/views/process_reader_integration_test.go b/go/views/process_reader_integration_test.go index 695e83d9..8d2d7b1f 100644 --- a/go/views/process_reader_integration_test.go +++ b/go/views/process_reader_integration_test.go @@ -7,6 +7,8 @@ import ( "encoding/json" "errors" "fmt" + "slices" + "strings" "testing" "time" @@ -156,7 +158,8 @@ func subTheViewsReaderAndTwoHandlersShareTheModulesOneDurable(t *testing.T) { t.Fatal(err) } } - r := &views.ProcessReader{Module: "reader", Hub: hub, Logf: t.Logf, + found := &seenFindings{put: make(chan map[string]any, 8)} + r := &views.ProcessReader{Module: "reader", Hub: hub, Logf: t.Logf, Findings: func() views.WireWriter { return found }, Call: func(tool string, body any) (json.RawMessage, error) { raw, _ := json.Marshal(body) m, err := nc.Request("mesh.mod.o.call.o_read_modules.n0", raw, 5*time.Second) @@ -168,6 +171,20 @@ func subTheViewsReaderAndTwoHandlersShareTheModulesOneDurable(t *testing.T) { if _, err := r.Follow("o", "modules", "o.o_read_modules"); err != nil { t.Fatal(err) } + // H-A-166: a seat's event said on its bare subject (as the seat mesh-catalog says registration-changed) and one with + // further parts both reach the one adopted handler, each once. + if info, err := js.Stream(ctx, process.EventsStream); err != nil { + t.Fatal(err) + } else if cfg := info.CachedInfo().Config; !slices.Contains(cfg.Subjects, "mesh.seat.*.event.>") { + cfg.Subjects = append(cfg.Subjects, "mesh.seat.*.event.>") + if _, err := js.UpdateStream(ctx, cfg); err != nil { + t.Fatal(err) + } + } + seatHeard := make(chan string, 8) + if err := r.AdoptEvent("s", "registration-changed", func(_ context.Context, d []byte) error { seatHeard <- string(d); return nil }); err != nil { + t.Fatal(err) + } r.Start(ctx, js, cred, nil) if err := hub.Start(ctx, js, cred); err != nil { t.Fatal(err) @@ -181,8 +198,9 @@ func subTheViewsReaderAndTwoHandlersShareTheModulesOneDurable(t *testing.T) { if err != nil { t.Fatal(err) } - for _, s := range []string{"mesh.mod.p.event.a", "mesh.mod.q.event.b.c", "mesh.mod.z.event.nobody", "mesh.mod.p.event.a"} { - if _, err := js.Publish(ctx, s, []byte("{}")); err != nil { + for _, s := range []string{"mesh.mod.p.event.a", "mesh.mod.q.event.b.c", "mesh.mod.z.event.nobody", "mesh.mod.p.event.a", + "mesh.seat.s.event.registration-changed", "mesh.seat.s.event.registration-changed.m"} { + if _, err := js.Publish(ctx, s, []byte(`"`+s+`"`)); err != nil { t.Fatal(err) } } @@ -199,8 +217,18 @@ func subTheViewsReaderAndTwoHandlersShareTheModulesOneDurable(t *testing.T) { t.Fatalf("handler %s got %s, wanted %s", want.who, s, want.subject) } } - if len(got["a"])+len(got["b"]) != 0 { - t.Fatal("a handler heard a message that was another's") + for _, want := range []string{`"mesh.seat.s.event.registration-changed"`, `"mesh.seat.s.event.registration-changed.m"`} { + select { + case d := <-seatHeard: + if d != want { + t.Fatalf("the adopted seat event heard %s, wanted %s", d, want) + } + case <-time.After(10 * time.Second): + t.Fatalf("the adopted seat event never heard %s", want) + } + } + if len(got["a"])+len(got["b"])+len(seatHeard) != 0 { + t.Fatal("a handler heard a message that was another's, or one twice") } cons, err := js.Consumer(ctx, process.EventsStream, cred.EventConsumerName()) if err != nil { @@ -209,6 +237,38 @@ func subTheViewsReaderAndTwoHandlersShareTheModulesOneDurable(t *testing.T) { if info, err := cons.Info(ctx); err != nil || info.NumAckPending != 0 { t.Fatalf("a message was left unacknowledged: %+v %v", info, err) } + // H-A-169: the one event no handler takes is the module's finding event-unhandled, counted once, naming its subject, + // though the reader started twice. + r.Start(ctx, js, cred, nil) + select { + case f := <-found.put: + summary, _ := f["summary"].(string) + if f["kind"] != views.EventUnhandledKind || f["id"] != "reader" || !strings.Contains(summary, "took 1 events") || + !strings.Contains(summary, "mesh.mod.z.event.nobody") { + t.Fatalf("the unhandled event's finding: %v", f) + } + case <-time.After(10 * time.Second): + t.Fatal("the unhandled event wrote no finding") + } + if _, err := js.Publish(ctx, "mesh.mod.z.event.nobody", []byte("{}")); err != nil { + t.Fatal(err) + } + if _, err := js.Publish(ctx, "mesh.mod.p.event.a", []byte("{}")); err != nil { + t.Fatal(err) + } + select { + case <-got["a"]: // the hub hands messages in order: every hook of the event before has run + case <-time.After(10 * time.Second): + t.Fatal("the handled event after the unhandled one was not heard") + } + select { + case f := <-found.put: + if summary, _ := f["summary"].(string); !strings.Contains(summary, "took 2 events") || len(found.put) != 0 { + t.Fatalf("after a second start one event is counted once: %v, and %d more", f, len(found.put)) + } + default: + t.Fatal("the unhandled event after the second start wrote no finding") + } // A hub started before the bus holds its durable says so, and binds at a later start: a module is not lost for the // order in which the bus makes its consumer. late := process.Account{Node: "n1", Module: "late"} @@ -244,3 +304,10 @@ func TestAReaderHearsAnOwnersChangesAsOrdinaryEvents(t *testing.T) { t.Run("TheViewsReaderAndTwoHandlersShareTheModulesOneDurable", subTheViewsReaderAndTwoHandlersShareTheModulesOneDurable) t.Run("WriterSaysItsEventsAndHearsAsksOnTheBus", subWriterSaysItsEventsAndHearsAsksOnTheBus) } + +// seenFindings is a findings writer that hands each finding written to the test. +type seenFindings struct{ put chan map[string]any } + +func (f *seenFindings) WriteWire(_ context.Context, w map[string]any) error { f.put <- w; return nil } +func (f *seenFindings) Clear(context.Context, string, string, string, string) error { return nil } +func (f *seenFindings) ModuleNode() (string, string) { return "reader", "n1" } diff --git a/go/views/reader.go b/go/views/reader.go index 9b37efcd..a2ca5c5d 100644 --- a/go/views/reader.go +++ b/go/views/reader.go @@ -74,6 +74,10 @@ type adoption struct { // further part, `..>` (design 47 G §532, §521). func (a adoption) local() string { return a.seat + "." + a.event + ".>" } +// bare is the adopted follower's event on its bare subject, `.`, heard by the same handler as local: the seat's +// contract says the event on either form (design 47 H, H-A-166). +func (a adoption) bare() string { return a.seat + "." + a.event } + func (r *Reader) logf(format string, args ...any) { if r.Logf != nil { r.Logf(format, args...) @@ -316,17 +320,23 @@ func (r *Reader) begin(name string) { r.asking[pattern] = true ctx, f := r.ctx, r.followers[name] handle := r.handler(ctx, pattern) + forms := []string{pattern} + if a, ok := r.adopted[name]; ok { + forms = []string{a.bare(), pattern} // one handler, both forms (H-A-166) + } go func() { - for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { - err := subscribe.get()(pattern, handle) - if err == nil { - break - } - r.logf("not hearing %s yet for %s's %s (%v); asking again in %s", pattern, f.Owner, f.State, err, wait) - select { - case <-ctx.Done(): - return - case <-time.After(wait): + for _, form := range forms { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + err := subscribe.get()(form, handle) + if err == nil { + break + } + r.logf("not hearing %s yet for %s's %s (%v); asking again in %s", form, f.Owner, f.State, err, wait) + select { + case <-ctx.Done(): + return + case <-time.After(wait): + } } } r.mu.Lock() diff --git a/go/views/views_test.go b/go/views/views_test.go index 67dde4f8..031b44c7 100644 --- a/go/views/views_test.go +++ b/go/views/views_test.go @@ -888,13 +888,17 @@ func subViews(t *testing.T) { } ready(next) hmu.Lock() - h, n := handlers["s.revision-said.>"], len(handlers) + h, bare, n := handlers["s.revision-said.>"], handlers["s.revision-said"], len(handlers) hmu.Unlock() - if h == nil || n != 1 { - t.Fatalf("subscriptions %v: want the event's one", handlers) + if h == nil || bare == nil || n != 2 { + t.Fatalf("subscriptions %v: want the event's bare subject and its further parts (H-A-166)", handlers) + } + // H-A-166: the event on its bare subject, as a seat's contract says it, reaches the same handler + if _ = bare(stdio.Envelope{Key: "s.revision-said", Body: []byte(`{"revision":7}`)}); len(*heard["main"]) != 1 || len(*heard["next"]) != 1 { + t.Errorf("the bare event reached %v and %v", *heard["main"], *heard["next"]) } event := stdio.Envelope{Key: "s.revision-said.next", Body: []byte(`{"revision":8}`)} - if _ = h(event); len(*heard["main"]) != 1 || len(*heard["next"]) != 1 { + if _ = h(event); len(*heard["main"]) != 2 || len(*heard["next"]) != 2 { t.Errorf("the event reached %v and %v", *heard["main"], *heard["next"]) } next.WaitFor(ctx, "9") @@ -911,7 +915,7 @@ func subViews(t *testing.T) { if st := r.Statuses(); len(st) != 1 || st[0].State != main.State || r.Follows("s."+next.State) { t.Errorf("first views after the drop: %+v", st) } - if _ = h(event); len(*heard["next"]) != 1 || len(*heard["main"]) != 2 { + if _ = h(event); len(*heard["next"]) != 2 || len(*heard["main"]) != 3 { t.Errorf("after the drop the event reached %v and %v", *heard["main"], *heard["next"]) } if err := r.Drop("s." + next.State); err == nil { @@ -932,7 +936,8 @@ func subViews(t *testing.T) { t.Fatal(err) } ready(pn) - if !slices.Contains(pr.Hub.Patterns(), "mesh.seat.s.event.revision-other.>") { + if ps := pr.Hub.Patterns(); !slices.Contains(ps, "mesh.seat.s.event.revision-other.>") || !slices.Contains(ps, "mesh.seat.s.event.revision-other") || + !slices.Contains(ps, "mesh.seat.s.event.revision-said") { t.Errorf("the hub handles %v", pr.Hub.Patterns()) } msg := func() *seatMsg { @@ -950,8 +955,23 @@ func subViews(t *testing.T) { t.Fatal(err) } m := msg() - if pr.Hear(ctx)(m); len(pfs.m) != 0 || len(pr.Followers()) != 1 || len(*heard["pnext"]) != 1 || !m.acked { - t.Errorf("after the drop: findings %v, followers %d, heard %v, acked %v", pfs.m, len(pr.Followers()), *heard["pnext"], m.acked) + if pr.Hear(ctx)(m); len(pr.Followers()) != 1 || len(*heard["pnext"]) != 1 || !m.acked { + t.Errorf("after the drop: followers %d, heard %v, acked %v", len(pr.Followers()), *heard["pnext"], m.acked) + } + // H-A-166, H-A-169: the event no handler takes, the dropped view's here, is the module's one finding event-unhandled, + // counted since the start, and the next start clears it. + if fd := pfs.next(t); fd.Kind != EventUnhandledKind || fd.ID != "reader" || !strings.Contains(fd.Summary, "took 1 events") || + !strings.Contains(fd.Summary, m.subject) || len(pfs.m) != 1 { + t.Errorf("after the drop: findings %v, the unhandled event's %+v", pfs.m, fd) + } + pr.Hear(ctx)(&seatMsg{subject: "mesh.seat.s.event.nobody"}) + if fd := pfs.next(t); !strings.Contains(fd.Summary, "took 2 events") || !strings.Contains(fd.Summary, "mesh.seat.s.event.nobody") || len(pfs.m) != 1 { + t.Errorf("the second unhandled event: findings %v, %+v", pfs.m, fd) + } + again := &ProcessReader{Module: "reader", Findings: func() WireWriter { return pfs }, Now: now, Hub: process.NewEventHub(t.Logf)} + again.Start(ctx, nil, process.Account{}, nil) + if len(pfs.m) != 0 { + t.Errorf("the start left the finding %v", pfs.m) } }) // Design 47 G §717: a view found by its seat starts without a holder, waits naming the seat, follows the holder the