From 380d4781e66f89887af3edea84314903d2f94f4f Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 15:42:24 +0100 Subject: [PATCH] mesh-grants: bring a held kept stream to its definition's source filter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The kept stream of the runner's pipeline-ended was made at G's definition, sourced by a three-part subject; H1 moved the event to ., so the stream took nothing in since 08:13 and the assigner, which hears a kept event from its kept stream, heard no end. createKept left every held kept stream as it was. It now brings a held one's source filter from the events stream to the definition's in place (a stream update, its messages and other settings kept, never made again) and says the held and the composed filter in the journal and the pass; the same holds for pipeline-started and wait-held. Builds: design 47 H H-A-145 (hq design/47-H1-answers-59 0787704b), §746 --- modules/mesh-grants/cmd/mesh-grants/create.go | 28 ++++++++ .../cmd/mesh-grants/integration_test.go | 29 ++++++++ modules/mesh-grants/grants_test.go | 11 +-- .../mesh-grants/internal/buscompose/nats.go | 3 +- .../internal/grants/askstreams_test.go | 3 + .../mesh-grants/internal/grants/assertions.go | 4 ++ .../internal/grants/grantskept_test.go | 70 +++++++++++++++++++ modules/mesh-grants/internal/grants/kept.go | 20 ++++-- modules/mesh-grants/internal/grants/part.go | 12 +++- 9 files changed, 169 insertions(+), 11 deletions(-) diff --git a/modules/mesh-grants/cmd/mesh-grants/create.go b/modules/mesh-grants/cmd/mesh-grants/create.go index a072cc46..eb2445ce 100644 --- a/modules/mesh-grants/cmd/mesh-grants/create.go +++ b/modules/mesh-grants/cmd/mesh-grants/create.go @@ -110,6 +110,34 @@ func (c creating) RaiseMaxBytes(stream string, to int64) (int64, bool, error) { return from, true, nil } +// SourceFilter brings the filter subject of a held stream's source from `source` to `filter` and changes nothing else +// (design 47 H H-A-145): it reads the stream's configuration and updates with the rest as it was, so its messages stay. A +// stream with no source from `source`, or one already at `filter`, is left as it is. +func (c creating) SourceFilter(stream, source, filter string) (string, bool, error) { + ctx, cancel := context.WithTimeout(c.ctx, requestWithin) + defer cancel() + s, err := c.js.Stream(ctx, stream) + if err != nil { + return "", false, err + } + cfg := s.CachedInfo().Config + i := slices.IndexFunc(cfg.Sources, func(src *jetstream.StreamSource) bool { return src != nil && src.Name == source }) + if i < 0 || cfg.Sources[i].FilterSubject == filter { + if i < 0 { + return "", false, nil + } + return cfg.Sources[i].FilterSubject, false, nil + } + from := cfg.Sources[i].FilterSubject + src := *cfg.Sources[i] + src.FilterSubject = filter + cfg.Sources[i] = &src + if _, err := c.js.UpdateStream(ctx, cfg); err != nil { + return from, false, err + } + return from, true, nil +} + // AddFilters adds filter subjects to a held consumer and changes nothing else. func (c creating) AddFilters(stream, name string, add []string) error { ctx, cancel := context.WithTimeout(c.ctx, requestWithin) diff --git a/modules/mesh-grants/cmd/mesh-grants/integration_test.go b/modules/mesh-grants/cmd/mesh-grants/integration_test.go index 225309db..a7d41915 100644 --- a/modules/mesh-grants/cmd/mesh-grants/integration_test.go +++ b/modules/mesh-grants/cmd/mesh-grants/integration_test.go @@ -611,6 +611,35 @@ func TestTheGrantsPartWhole(t *testing.T) { if again, _ := stream.Info(ctx); !slices.Equal(again.Config.Subjects, []string{"a.>", "b.>"}) { t.Errorf("subjects after adding a held one: %v", again.Config.Subjects) } + // A kept stream's source filter is brought to its definition's in place: the stream is not made again, its settings + // stay, and an equal filter or a source of another stream changes nothing (design 47 H H-A-145, §746). + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "KEPTSRC", Subjects: []string{"ev.>"}}); err != nil { + t.Fatal(err) + } + kept, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "KEPT", MaxAge: time.Hour, + Sources: []*jetstream.StreamSource{{Name: "KEPTSRC", FilterSubject: "ev.ended.*.*.*"}}}) + if err != nil { + t.Fatal(err) + } + keptBefore, _ := kept.Info(ctx) + for _, row := range []struct { + source, filter, from, holds string + changed bool + }{ + {"KEPTSRC", "ev.ended.*.*", "ev.ended.*.*.*", "ev.ended.*.*", true}, + {"KEPTSRC", "ev.ended.*.*", "ev.ended.*.*", "ev.ended.*.*", false}, + {"OTHER", "ev.other", "", "ev.ended.*.*", false}, + } { + from, changed, err := c.SourceFilter("KEPT", row.source, row.filter) + if err != nil { + t.Fatal(err) + } + now, _ := kept.Info(ctx) + if from != row.from || changed != row.changed || len(now.Config.Sources) != 1 || now.Config.Sources[0].FilterSubject != row.holds || + now.Config.MaxAge != keptBefore.Config.MaxAge || !now.Created.Equal(keptBefore.Created) { + t.Errorf("source %s to %s: from %q changed %v, config %+v created %v", row.source, row.filter, from, changed, now.Config, now.Created) + } + } // The named asks' purge takes one subject's messages and no other's, and a named worker's delete is said done also // where the bus holds none (design 47 G §544). for _, subj := range []string{"b.one", "b.two"} { diff --git a/modules/mesh-grants/grants_test.go b/modules/mesh-grants/grants_test.go index 1e45fdc3..68aa6546 100644 --- a/modules/mesh-grants/grants_test.go +++ b/modules/mesh-grants/grants_test.go @@ -202,6 +202,9 @@ func (b *fakeBus) AddSubjects(stream string, add []string) error { return errors.New("stream not found") } +// SourceFilter is the bus's update of a held stream's source filter: the fake holds no sources, so nothing changes. +func (b *fakeBus) SourceFilter(string, string, string) (string, bool, error) { return "", false, nil } + // RaiseMaxBytes is the bus's raise of a held stream's size cap: only to a larger cap, an unbounded stream and every other // setting kept. func (b *fakeBus) RaiseMaxBytes(stream string, to int64) (int64, bool, error) { @@ -684,10 +687,10 @@ func TestGrants(t *testing.T) { if n := strings.Count(string(src), "CreateStream"); (path == createOnly && n != 1) || (path != createOnly && n != 0) { t.Errorf("%s holds CreateStream %d times: only create.go creates, once", path, n) } - // The two updates of a stream: adding subjects to a held one (the operator, 2026-11-05) and raising its size cap - // (2026-11-09), both in create.go. - if n := strings.Count(string(src), "UpdateStream("); (path == createOnly && n != 2) || (path != createOnly && n != 0) { - t.Errorf("%s holds UpdateStream %d times: only create.go updates a stream, twice, to add subjects and to raise a size cap", path, n) + // The three updates of a stream: adding subjects to a held one (the operator, 2026-11-05), raising its size cap + // (2026-11-09) and bringing a kept stream's source filter to its definition's (design 47 H H-A-145), all in create.go. + if n := strings.Count(string(src), "UpdateStream("); (path == createOnly && n != 3) || (path != createOnly && n != 0) { + t.Errorf("%s holds UpdateStream %d times: only create.go updates a stream, three times, to add subjects, to raise a size cap and to bring a kept stream's source filter", path, n) } // The one object bucket: the part's own, of the user list sealed to the bus node, in sealed.go (ADR 0547 clause 7). if path == sealedOnly { diff --git a/modules/mesh-grants/internal/buscompose/nats.go b/modules/mesh-grants/internal/buscompose/nats.go index a62f8c81..bd963cb0 100644 --- a/modules/mesh-grants/internal/buscompose/nats.go +++ b/modules/mesh-grants/internal/buscompose/nats.go @@ -1765,7 +1765,8 @@ const WatchmanModule = "mesh-delivery-watchman" // 2026-10-26 08:54): its create, and its information to read back what it holds. Each by its own name: a name that // is not one plain subject token is left out, since a wildcard there would be every stream. Its update is granted for the one // acts the part does on a held stream: adding the subjects a manifest's new events need (grants.Creator.AddSubjects) and -// raising its size cap to the manifest's when larger (grants.Creator.RaiseMaxBytes; the operator, 2026-11-09); it never +// raising its size cap to the manifest's when larger (grants.Creator.RaiseMaxBytes; the operator, 2026-11-09), and bringing +// a kept stream's source filter to its definition's (grants.Creator.SourceFilter; design 47 H H-A-145); it never // removes a subject, lowers a limit or recreates the stream, and never deletes or purges one. func CreateStreamGrants(streams []string) []string { var out []string diff --git a/modules/mesh-grants/internal/grants/askstreams_test.go b/modules/mesh-grants/internal/grants/askstreams_test.go index cac2c39d..bb070dda 100644 --- a/modules/mesh-grants/internal/grants/askstreams_test.go +++ b/modules/mesh-grants/internal/grants/askstreams_test.go @@ -21,6 +21,9 @@ func (c *askCreator) AddSubjects(string, []string) error { return nil } func (c *askCreator) RaiseMaxBytes(string, int64) (int64, bool, error) { return 0, false, nil } +func (c *askCreator) SourceFilter(string, string, string) (string, bool, error) { + return "", false, nil +} // A seat whose queues came with G (mesh-builds) gets its work-queue ask stream from the seat list, as the controller made one for // every seat before G; a seat that accepts nothing gets none, and a stream the bus holds is never made again (lead-47, 2026-11-15). diff --git a/modules/mesh-grants/internal/grants/assertions.go b/modules/mesh-grants/internal/grants/assertions.go index 3de82b57..d3fb0dc7 100644 --- a/modules/mesh-grants/internal/grants/assertions.go +++ b/modules/mesh-grants/internal/grants/assertions.go @@ -69,6 +69,10 @@ type Creator interface { // changes nothing else (the operator, 2026-11-09 09:07: "raise only"). It answers the cap it held and whether it // raised; a cap that is larger or equal, or an unbounded stream, is left as it is. It never lowers any limit. RaiseMaxBytes(stream string, to int64) (from int64, raised bool, err error) + // SourceFilter brings the filter subject of a held stream's source from the stream `source` to `filter`, keeping its + // messages and every other setting, the stream not made again (design 47 H H-A-145). It answers the filter it held and + // whether it changed; a stream with no source from `source` is left as it is. + SourceFilter(stream, source, filter string) (from string, changed bool, err error) } // Raise is one held stream's size cap raised to the manifest's, said in the event and the journal. diff --git a/modules/mesh-grants/internal/grants/grantskept_test.go b/modules/mesh-grants/internal/grants/grantskept_test.go index 92740e6f..a7e66e66 100644 --- a/modules/mesh-grants/internal/grants/grantskept_test.go +++ b/modules/mesh-grants/internal/grants/grantskept_test.go @@ -1,8 +1,11 @@ package grants import ( + "strings" "testing" + "github.com/nats-io/nats.go/jetstream" + "mesh-grants/internal/buscompose" ) @@ -39,3 +42,70 @@ func TestTheGrantsAccountMayCreateEachKeptEventStream(t *testing.T) { }) } } + +// sourcing is a Creator holding kept streams by their source filter and message count: SourceFilter updates a held one in +// place, so its messages stay; CreateDeclared makes one the bus lacks. +type sourcing struct { + Creator + filters map[string]string // by stream, its source filter from EVENTS + messages map[string]int + made []string +} + +func (s *sourcing) SourceFilter(stream, source, filter string) (string, bool, error) { + from, ok := s.filters[stream] + if !ok || source != buscompose.EventsStream || from == filter { + return from, false, nil + } + s.filters[stream] = filter + return from, true, nil +} + +func (s *sourcing) CreateDeclared(cfg jetstream.StreamConfig) error { + s.made = append(s.made, cfg.Name) + s.filters[cfg.Name], s.messages[cfg.Name] = cfg.Sources[0].FilterSubject, 0 + return nil +} + +// H-A-145 (design 47 H §746): a held kept stream whose source filter the definition changed is brought to the definition's +// filter in place, its messages kept and the change said with both filters; one at the definition's filter is untouched; one +// the bus lacks is made. +func TestAKeptStreamFollowsItsDefinitionsSourceFilter(t *testing.T) { + const ended = "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.*.*" + for _, row := range []struct { + name string + held string // the held stream's source filter; "" for a stream the bus lacks + made bool + moved string + filter string + messages, keptMsgs int + }{ + {name: "held at the definition's filter: untouched", held: ended, filter: ended, messages: 604, keptMsgs: 604}, + {name: "held at an earlier filter: brought to the definition's, messages kept", held: ended + ".*", filter: ended, messages: 604, keptMsgs: 604, + moved: "SEAT_mesh-pipeline-runner_pipeline-ended: source filter " + ended + ".* brought to " + ended + ", its messages kept"}, + {name: "not held: made", made: true, filter: ended}, + } { + t.Run(row.name, func(t *testing.T) { + k := buscompose.KeptEvent{Seat: "mesh-pipeline-runner", Event: "pipeline-ended", Stream: "SEAT_mesh-pipeline-runner_pipeline-ended", Filter: ended, MaxAgeDays: 90, MaxMiB: 192} + c := &sourcing{filters: map[string]string{}, messages: map[string]int{}} + held := map[string]Limits{} + if row.held != "" { + c.filters[k.Stream], c.messages[k.Stream] = row.held, row.messages + held[k.Stream] = Limits{} + } + made, moved, notMade := createKept(c, &creations{}, []buscompose.KeptEvent{k}, held, false) + if len(notMade) != 0 { + t.Fatalf("not made: %v", notMade) + } + if (len(made) == 1) != row.made || len(c.made) != len(made) { + t.Errorf("made %v (bus %v), want made %v", made, c.made, row.made) + } + if got := strings.Join(moved, "; "); got != row.moved { + t.Errorf("moved %q, want %q", got, row.moved) + } + if c.filters[k.Stream] != row.filter || c.messages[k.Stream] != row.keptMsgs { + t.Errorf("the stream holds filter %q and %d messages, want %q and %d", c.filters[k.Stream], c.messages[k.Stream], row.filter, row.keptMsgs) + } + }) + } +} diff --git a/modules/mesh-grants/internal/grants/kept.go b/modules/mesh-grants/internal/grants/kept.go index 2cad491a..ebbb3f54 100644 --- a/modules/mesh-grants/internal/grants/kept.go +++ b/modules/mesh-grants/internal/grants/kept.go @@ -65,14 +65,24 @@ func keptOutOfHearing(kept []buscompose.KeptEvent, watches []buscompose.Seat, co } // createKept makes each kept event's stream the bus lacks, from the shared events stream's messages of its subject, and -// says what it made. A stream the bus holds is left as it is; one made at this seat list and gone since is made again only on -// a hand assert (the same rule as a manifest's streams: made once per definition). held is what the bus held at this pass. -func createKept(c Creator, done *creations, kept []buscompose.KeptEvent, held map[string]Limits, force bool) (made, notMade []string) { +// says what it made. A stream the bus holds keeps its messages and is not made again; its source filter is brought to the +// definition's in place when the definition changed the event's subject (design 47 H H-A-145, §746: the runner's +// pipeline-ended moved from three parts to two and its kept stream took nothing in), said in moved with the held and the +// composed filter. One made at this seat list and gone since is made again only on a hand assert (the same rule as a +// manifest's streams: made once per definition). held is what the bus held at this pass. +func createKept(c Creator, done *creations, kept []buscompose.KeptEvent, held map[string]Limits, force bool) (made, moved, notMade []string) { for _, k := range kept { + cfg := k.StreamConfig() if _, ok := held[k.Stream]; ok { + from, changed, err := c.SourceFilter(k.Stream, buscompose.EventsStream, k.Filter) + switch { + case err != nil: + notMade = append(notMade, k.Stream+": its source filter is not brought to "+k.Filter+": "+err.Error()+" (tried again at the next pass)") + case changed: + moved = append(moved, k.Stream+": source filter "+from+" brought to "+k.Filter+", its messages kept") + } continue } - cfg := k.StreamConfig() digest := fmt.Sprintf("%v", cfg.Sources[0].FilterSubject) + fmt.Sprint(k.Last, k.MaxAgeDays, k.MaxMiB) if !force && done.madeAt(k.Stream) == digest { notMade = append(notMade, k.Stream+": made at this definition and gone since; grants_assert makes it again") @@ -87,7 +97,7 @@ func createKept(c Creator, done *creations, kept []buscompose.KeptEvent, held ma notMade = append(notMade, k.Stream+": "+err.Error()+" (tried again at the next event)") } } - return made, notMade + return made, moved, notMade } // eventSubjectsOf is, by seat, each event's subject pattern (`running..`) of the events whose subject carries diff --git a/modules/mesh-grants/internal/grants/part.go b/modules/mesh-grants/internal/grants/part.go index 1b94c900..2ee46a4f 100644 --- a/modules/mesh-grants/internal/grants/part.go +++ b/modules/mesh-grants/internal/grants/part.go @@ -565,11 +565,21 @@ func (p *Part) Compose() Result { } forced := ask.Assert if p.Create != nil && heldErr == nil { - made, notMade := createKept(p.Create, &p.made, in.Kept, held, forced != "") + made, moved, notMade := createKept(p.Create, &p.made, in.Kept, held, forced != "") if len(made) > 0 { res.Asserted = append(res.Asserted, "the seats' kept events: created "+strings.Join(made, ", ")) p.say(&res, EventAsserted, map[string]any{"module": Module, "created": made}) } + for _, m := range moved { + // Said in the journal with both filters (H-A-145). + res.Asserted = append(res.Asserted, "the seats' kept events: "+m) + if p.Logf != nil { + p.Logf("kept stream %s", m) + } + } + if len(moved) > 0 { + p.say(&res, EventAsserted, map[string]any{"module": Module, "source-filters": moved}) + } for _, why := range notMade { res.NotAsserted = append(res.NotAsserted, "kept event "+why) }