From 70bb8f7f5152d0809f21a46d2c641779917c6987 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 19:21:24 +0100 Subject: [PATCH 1/2] Teller test bed: answer who holds operator-channel, so a catch-up does not hold the asks' watch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The want-of-room test failed most runs since 20 Nov: the teller's picture of the assigner's asks hands the first ask on twice (its first view, then again), and the second sight of an open asking catches up with the router's record, which first asks the seat mesh-module-assigner's assignments who holds operator-channel. The bed neither granted nor answered that call, so it waited out its call timeout on the watch's goroutine, and the second ask, written meanwhile, was not raised within the test's 10 s. The bed now grants and answers the call as the live seat does, naming ops-router, so the catch-up reads ops-router_asks at once (20 of 20 runs pass, were 0 of 20 locally). issue 1960 (H1), test stability. Builds: design 47 G §570 (issue 1610 AC1, the refused ask asked again), §717 --- .../cmd/mesh-delivery-teller/integration_test.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/integration_test.go b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/integration_test.go index 059a5628..1b046069 100644 --- a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/integration_test.go +++ b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/integration_test.go @@ -120,6 +120,10 @@ func fullConf(dir string) string { teller = append(append(teller, viewstest.Grants(assignerModule, assignerModule+".seat-holders", tellerConsumer)...), "mesh.seat.mesh-catalog.call.revision", "mesh.seat.mesh-catalog.call.revision.>", "mesh.seat.mesh-catalog.call.registrations", "mesh.seat.mesh-catalog.call.registrations.>", "mesh.seat.mesh-delivery.call.source", "mesh.seat.mesh-delivery.call.source.>") + // Who holds operator-channel, asked of the seat mesh-module-assigner's assignments (manifest invokes) before the router's + // record of an open ask is read: a bed that refused the call held the teller's watch of the asks for the call's whole + // timeout at each catch-up, and every ask written meanwhile waited (issue 1960, H1). + teller = append(teller, "mesh.seat.mesh-module-assigner.call.assignments", "mesh.seat.mesh-module-assigner.call.assignments.>") teller = append(teller, keepsOf(bucketsOf(tellerModule).findings)...) view := append(append(readOf(tellerAnswers), readOf(assignerAsks)...), keepsOf(viewAnswers)...) assigner := keepsOf(assignerAsks) @@ -224,7 +228,9 @@ func (b *fullBed) place(t *testing.T) { } // serveCatalogHalf answers the catalog half's three first views (design 47 G §614): the gate's one source branch, the -// assigner registered on it, and its trunk revision whose entry of the assigner claims mesh-module-assigner. +// assigner registered on it, and its trunk revision whose entry of the assigner claims mesh-module-assigner. It also answers +// the seat mesh-module-assigner's assignments as the live seat does, naming ops-router the holder of operator-channel, so a +// catch-up reads the router's record (ops-router_asks) at once. func serveCatalogHalf(t *testing.T, nc *nats.Conn) { t.Helper() respond := func(a any) nats.MsgHandler { @@ -235,8 +241,10 @@ func serveCatalogHalf(t *testing.T, nc *nats.Conn) { Entries: []meshcatalog.CatalogEntry{{Module: assignerModule, ManifestDigest: "d1", Manifest: map[string]any{"module": assignerModule, "claims": []any{map[string]any{"name": "mesh-module-assigner", "scope": "mesh"}}}}}} registrations := meshcatalog.RegistrationsAnswer{Registrations: []meshcatalog.Registration{{Module: assignerModule, Version: 1, Branches: []string{"release"}}}} + assignments := json.RawMessage(`{"seats":[{"seat":"operator-channel","scope":"mesh","holders":[{"module":"ops-router","node":"n2"}]}]}`) for subject, handle := range map[string]nats.MsgHandler{"mesh.seat.mesh-catalog.call.revision": respond(revision), - "mesh.seat.mesh-catalog.call.registrations": respond(registrations), "mesh.seat.mesh-delivery.call.source": respond(map[string]any{"branches": []string{"release"}})} { + "mesh.seat.mesh-module-assigner.call.assignments": respond(assignments), + "mesh.seat.mesh-catalog.call.registrations": respond(registrations), "mesh.seat.mesh-delivery.call.source": respond(map[string]any{"branches": []string{"release"}})} { for _, s := range []string{subject, subject + ".>"} { sub, err := nc.Subscribe(s, handle) if err != nil { -- 2.54.0 From b79fd19c8484344562163c9d0b1c71301238fab4 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 19:24:22 +0100 Subject: [PATCH 2/2] Teller: hear registration-changed on the views reader's one subscription MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since mesh-sdk heard a seat's event on its bare subject too (H-A-166), the teller's own hub handler of registration-changed overlapped the one the views reader makes for the catalog half's followers: the hub refused the reader's, said "a message has one taker", and registered none of the reader's adopted events after it, so the catalog half's registrations and source branches were their first views' alone. The widenings now hear the event through ProcessReader.AdoptEvent beside the followers; a failed handling is given back for the bus to ask again, as before. The carry test now fails when any of the reader's events is refused. issue 1960 (H1), test stability. Builds: design 47 G A-199, A-222, §532, §335 --- .../mesh-delivery-teller/integration_test.go | 5 +++ .../cmd/mesh-delivery-teller/main.go | 9 +++-- .../cmd/mesh-delivery-teller/widens.go | 39 +++++++++++-------- .../cmd/mesh-delivery-teller/widens_test.go | 9 ++--- 4 files changed, 38 insertions(+), 24 deletions(-) diff --git a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/integration_test.go b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/integration_test.go index 1b046069..304549dd 100644 --- a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/integration_test.go +++ b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/integration_test.go @@ -383,6 +383,11 @@ func TestTellerCarriesAnAskAndWritesTheAnswer(t *testing.T) { // 1343/AC1, AC2 } // The program hears warrants before it reads the asks; it has its subscription before the ask is written. b.listening(t) + // Every event the views reader adopted is heard once the hub pulls: registration-changed, which the teller's widenings and the + // catalog half both hear, is one subscription (A-199, §532), so no handler of the teller takes the reader's away. + if l := logs.String(); strings.Contains(l, "a message has one taker") { + t.Errorf("an event of the views reader is not heard: two handlers took one subject\n%s", l) + } // Design 47 G §717: the teller started before any module held mesh-wants; once the holder is placed it follows it, and // raises the ask the holder then writes. b.place(t) diff --git a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/main.go b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/main.go index 9fa18d8d..7e55d7a9 100644 --- a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/main.go +++ b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/main.go @@ -13,6 +13,7 @@ import ( "git.novox.be/novox/mesh-contracts/go/seats/meshbroker" occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdeliveryview" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "io" @@ -91,9 +92,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err := hub.Handle(pipelineEndedSubject, widens.HearEnded); err != nil { return fmt.Errorf("hearing %s: %w", pipelineEndedSubject, err) } - if err := hub.Handle(registrationChangedSubject, widens.HearRegistration); err != nil { - return fmt.Errorf("hearing %s: %w", registrationChangedSubject, err) - } // A merge named a recovery is told on every channel (design 47 G §81, §514); a merge made at the forge is no record of its // own (H §763, H-A-119). if err := hub.Handle(recoveryMergedSubject, widens.HearRecovery); err != nil { @@ -108,6 +106,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return calls.Emit(answeredEvent, Answered{Key: key, Ask: askID, Answer: answer}, stdio.EmitOptions{MessageID: answeredID(key, askID)}) } reader := &views.ProcessReader{Module: cred.Module, Call: calls.Call, Logf: logger.Printf, Hub: hub} + // registration-changed is heard on the reader's one subscription of it, which the catalog client's followers share + // (A-199, §532): the hub gives a message one taker. + if err := reader.AdoptEvent(meshcatalog.Seat, meshcatalog.EventRegistrationChanged, widens.HearRegistration); err != nil { + return fmt.Errorf("hearing %s: %w", meshcatalog.SubjectRegistrationChanged, err) + } // The teller's finding stream is its `finding-events` entry, asked of the seat mesh-broker's show (design 47 G A-274, // §529, §577): the writer says its events under that binding's subjects. binding := process.NewBinding(meshbroker.Seat, cred.Module, calls.CallKept, logger.Printf, views.FindingEventsLocal) diff --git a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/widens.go b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/widens.go index 2a2e928d..4ba70f2e 100644 --- a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/widens.go +++ b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/widens.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "errors" "fmt" @@ -33,9 +34,8 @@ const ( // The events the teller hears for it (manifest consumes). const ( - pipelineEndedSubject = "mesh.seat." + mpr.Seat + ".event." + mpr.EventPipelineEnded + ".>" - registrationChangedSubject = meshcatalog.SubjectRegistrationChanged - recoveryMergedSubject = meshdelivery.SubjectRecoveryMerged + pipelineEndedSubject = "mesh.seat." + mpr.Seat + ".event." + mpr.EventPipelineEnded + ".>" + recoveryMergedSubject = meshdelivery.SubjectRecoveryMerged ) // widensState is the state widens-said as the teller keeps it. @@ -151,32 +151,39 @@ func (w *Widens) HearEnded(m jetstream.Msg) { } // HearRegistration is the handler of registration-changed: it asks the registration of the module and drops the branches it -// no longer watches. -func (w *Widens) HearRegistration(m jetstream.Msg) { +// no longer watches. It is heard on the one subscription the views reader's followers of the event share (ProcessReader's +// AdoptEvent, design 47 G A-199, §532), beside the catalog client's own handler: the event hub gives a message one taker, +// so a second handler of the subject on the hub would take the reader's subscription away. A failure leaves the message +// unacknowledged, and the bus asks again (A-222); a message that names no module is passed over. +func (w *Widens) HearRegistration(_ context.Context, data []byte) error { var ev meshcatalog.RegistrationChangedEvent - if err := json.Unmarshal(m.Data(), &ev); err != nil || ev.Module == "" { - w.logf("%s names no module and is left: %v", m.Subject(), err) - _ = m.Ack() - return + if err := json.Unmarshal(data, &ev); err != nil || ev.Module == "" { + w.logf("a registration-changed that names no module is left: %v", err) + return nil } raw, err := w.Call(registrationsKey, map[string]any{"module": ev.Module}) if err != nil { - w.finish(m, fmt.Errorf("the registration of %s cannot be read: %w", ev.Module, err)) - return + return w.heardAgain(fmt.Errorf("the registration of %s cannot be read: %w", ev.Module, err)) } var ans meshcatalog.RegistrationsAnswer if err := json.Unmarshal(raw, &ans); err != nil { - w.finish(m, fmt.Errorf("the registration of %s is not read: %w", ev.Module, err)) - return + return w.heardAgain(fmt.Errorf("the registration of %s is not read: %w", ev.Module, err)) } for _, r := range ans.Registrations { if r.Module == ev.Module { - w.finish(m, w.Unwatched(ev.Module, r.Branches)) - return + return w.heardAgain(w.Unwatched(ev.Module, r.Branches)) } } // A module with no registration watches no branch. - w.finish(m, w.Unwatched(ev.Module, nil)) + return w.heardAgain(w.Unwatched(ev.Module, nil)) +} + +// heardAgain says err, when not nil, as a message heard again, and gives it back. +func (w *Widens) heardAgain(err error) error { + if err != nil { + w.logf("%v; the message is heard again", err) + } + return err } // finish acknowledges the message when err is nil and asks it again when it is not. diff --git a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/widens_test.go b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/widens_test.go index 8d2f4072..37759835 100644 --- a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/widens_test.go +++ b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/widens_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "errors" "slices" @@ -155,15 +156,13 @@ func TestARegistrationChangedDropsTheBranchesNoLongerWatched(t *testing.T) { st.put(k, "c") } w := &Widens{State: st, Call: c.call} - m := &fakeMsg{data: []byte(`{"module":"nats","moved":[],"version":2,"warrant":"w"}`)} - w.HearRegistration(m) + if err := w.HearRegistration(context.Background(), []byte(`{"module":"nats","moved":[],"version":2,"warrant":"w"}`)); err != nil { + t.Errorf("the registration-changed is given back: %v", err) + } keys, _ := st.Keys() if want := []string{"nats.main", "natsx.next", "other.next"}; !slices.Equal(keys, want) { t.Errorf("keys %v, want %v", keys, want) } - if !m.acked || m.naked { - t.Errorf("acked %v naked %v", m.acked, m.naked) - } } // A pipeline-ended whose say fails is given back, and one that is told is acknowledged. -- 2.54.0