mesh-grants: bring a held kept stream to its definition's source filter (H-A-145) #269
@@ -110,6 +110,34 @@ func (c creating) RaiseMaxBytes(stream string, to int64) (int64, bool, error) {
|
||||
return from, true, nil
|
||||
}
|
||||
|
||||
// SourceFilter brings the filter subject of a held stream's source from `source` to `filter` and changes nothing else
|
||||
// (design 47 H H-A-145): it reads the stream's configuration and updates with the rest as it was, so its messages stay. A
|
||||
// stream with no source from `source`, or one already at `filter`, is left as it is.
|
||||
func (c creating) SourceFilter(stream, source, filter string) (string, bool, error) {
|
||||
ctx, cancel := context.WithTimeout(c.ctx, requestWithin)
|
||||
defer cancel()
|
||||
s, err := c.js.Stream(ctx, stream)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
cfg := s.CachedInfo().Config
|
||||
i := slices.IndexFunc(cfg.Sources, func(src *jetstream.StreamSource) bool { return src != nil && src.Name == source })
|
||||
if i < 0 || cfg.Sources[i].FilterSubject == filter {
|
||||
if i < 0 {
|
||||
return "", false, nil
|
||||
}
|
||||
return cfg.Sources[i].FilterSubject, false, nil
|
||||
}
|
||||
from := cfg.Sources[i].FilterSubject
|
||||
src := *cfg.Sources[i]
|
||||
src.FilterSubject = filter
|
||||
cfg.Sources[i] = &src
|
||||
if _, err := c.js.UpdateStream(ctx, cfg); err != nil {
|
||||
return from, false, err
|
||||
}
|
||||
return from, true, nil
|
||||
}
|
||||
|
||||
// AddFilters adds filter subjects to a held consumer and changes nothing else.
|
||||
func (c creating) AddFilters(stream, name string, add []string) error {
|
||||
ctx, cancel := context.WithTimeout(c.ctx, requestWithin)
|
||||
|
||||
@@ -611,6 +611,35 @@ func TestTheGrantsPartWhole(t *testing.T) {
|
||||
if again, _ := stream.Info(ctx); !slices.Equal(again.Config.Subjects, []string{"a.>", "b.>"}) {
|
||||
t.Errorf("subjects after adding a held one: %v", again.Config.Subjects)
|
||||
}
|
||||
// A kept stream's source filter is brought to its definition's in place: the stream is not made again, its settings
|
||||
// stay, and an equal filter or a source of another stream changes nothing (design 47 H H-A-145, §746).
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "KEPTSRC", Subjects: []string{"ev.>"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kept, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "KEPT", MaxAge: time.Hour,
|
||||
Sources: []*jetstream.StreamSource{{Name: "KEPTSRC", FilterSubject: "ev.ended.*.*.*"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
keptBefore, _ := kept.Info(ctx)
|
||||
for _, row := range []struct {
|
||||
source, filter, from, holds string
|
||||
changed bool
|
||||
}{
|
||||
{"KEPTSRC", "ev.ended.*.*", "ev.ended.*.*.*", "ev.ended.*.*", true},
|
||||
{"KEPTSRC", "ev.ended.*.*", "ev.ended.*.*", "ev.ended.*.*", false},
|
||||
{"OTHER", "ev.other", "", "ev.ended.*.*", false},
|
||||
} {
|
||||
from, changed, err := c.SourceFilter("KEPT", row.source, row.filter)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now, _ := kept.Info(ctx)
|
||||
if from != row.from || changed != row.changed || len(now.Config.Sources) != 1 || now.Config.Sources[0].FilterSubject != row.holds ||
|
||||
now.Config.MaxAge != keptBefore.Config.MaxAge || !now.Created.Equal(keptBefore.Created) {
|
||||
t.Errorf("source %s to %s: from %q changed %v, config %+v created %v", row.source, row.filter, from, changed, now.Config, now.Created)
|
||||
}
|
||||
}
|
||||
// The named asks' purge takes one subject's messages and no other's, and a named worker's delete is said done also
|
||||
// where the bus holds none (design 47 G §544).
|
||||
for _, subj := range []string{"b.one", "b.two"} {
|
||||
|
||||
@@ -202,6 +202,9 @@ func (b *fakeBus) AddSubjects(stream string, add []string) error {
|
||||
return errors.New("stream not found")
|
||||
}
|
||||
|
||||
// SourceFilter is the bus's update of a held stream's source filter: the fake holds no sources, so nothing changes.
|
||||
func (b *fakeBus) SourceFilter(string, string, string) (string, bool, error) { return "", false, nil }
|
||||
|
||||
// RaiseMaxBytes is the bus's raise of a held stream's size cap: only to a larger cap, an unbounded stream and every other
|
||||
// setting kept.
|
||||
func (b *fakeBus) RaiseMaxBytes(stream string, to int64) (int64, bool, error) {
|
||||
@@ -684,10 +687,10 @@ func TestGrants(t *testing.T) {
|
||||
if n := strings.Count(string(src), "CreateStream"); (path == createOnly && n != 1) || (path != createOnly && n != 0) {
|
||||
t.Errorf("%s holds CreateStream %d times: only create.go creates, once", path, n)
|
||||
}
|
||||
// The two updates of a stream: adding subjects to a held one (the operator, 2026-11-05) and raising its size cap
|
||||
// (2026-11-09), both in create.go.
|
||||
if n := strings.Count(string(src), "UpdateStream("); (path == createOnly && n != 2) || (path != createOnly && n != 0) {
|
||||
t.Errorf("%s holds UpdateStream %d times: only create.go updates a stream, twice, to add subjects and to raise a size cap", path, n)
|
||||
// The three updates of a stream: adding subjects to a held one (the operator, 2026-11-05), raising its size cap
|
||||
// (2026-11-09) and bringing a kept stream's source filter to its definition's (design 47 H H-A-145), all in create.go.
|
||||
if n := strings.Count(string(src), "UpdateStream("); (path == createOnly && n != 3) || (path != createOnly && n != 0) {
|
||||
t.Errorf("%s holds UpdateStream %d times: only create.go updates a stream, three times, to add subjects, to raise a size cap and to bring a kept stream's source filter", path, n)
|
||||
}
|
||||
// The one object bucket: the part's own, of the user list sealed to the bus node, in sealed.go (ADR 0547 clause 7).
|
||||
if path == sealedOnly {
|
||||
|
||||
@@ -1765,7 +1765,8 @@ const WatchmanModule = "mesh-delivery-watchman"
|
||||
// 2026-10-26 08:54): its create, and its information to read back what it holds. Each by its own name: a name that
|
||||
// is not one plain subject token is left out, since a wildcard there would be every stream. Its update is granted for the one
|
||||
// acts the part does on a held stream: adding the subjects a manifest's new events need (grants.Creator.AddSubjects) and
|
||||
// raising its size cap to the manifest's when larger (grants.Creator.RaiseMaxBytes; the operator, 2026-11-09); it never
|
||||
// raising its size cap to the manifest's when larger (grants.Creator.RaiseMaxBytes; the operator, 2026-11-09), and bringing
|
||||
// a kept stream's source filter to its definition's (grants.Creator.SourceFilter; design 47 H H-A-145); it never
|
||||
// removes a subject, lowers a limit or recreates the stream, and never deletes or purges one.
|
||||
func CreateStreamGrants(streams []string) []string {
|
||||
var out []string
|
||||
|
||||
@@ -21,6 +21,9 @@ func (c *askCreator) AddSubjects(string, []string) error { return nil }
|
||||
func (c *askCreator) RaiseMaxBytes(string, int64) (int64, bool, error) {
|
||||
return 0, false, nil
|
||||
}
|
||||
func (c *askCreator) SourceFilter(string, string, string) (string, bool, error) {
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
// A seat whose queues came with G (mesh-builds) gets its work-queue ask stream from the seat list, as the controller made one for
|
||||
// every seat before G; a seat that accepts nothing gets none, and a stream the bus holds is never made again (lead-47, 2026-11-15).
|
||||
|
||||
@@ -69,6 +69,10 @@ type Creator interface {
|
||||
// changes nothing else (the operator, 2026-11-09 09:07: "raise only"). It answers the cap it held and whether it
|
||||
// raised; a cap that is larger or equal, or an unbounded stream, is left as it is. It never lowers any limit.
|
||||
RaiseMaxBytes(stream string, to int64) (from int64, raised bool, err error)
|
||||
// SourceFilter brings the filter subject of a held stream's source from the stream `source` to `filter`, keeping its
|
||||
// messages and every other setting, the stream not made again (design 47 H H-A-145). It answers the filter it held and
|
||||
// whether it changed; a stream with no source from `source` is left as it is.
|
||||
SourceFilter(stream, source, filter string) (from string, changed bool, err error)
|
||||
}
|
||||
|
||||
// Raise is one held stream's size cap raised to the manifest's, said in the event and the journal.
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
package grants
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"mesh-grants/internal/buscompose"
|
||||
)
|
||||
|
||||
@@ -39,3 +42,70 @@ func TestTheGrantsAccountMayCreateEachKeptEventStream(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// sourcing is a Creator holding kept streams by their source filter and message count: SourceFilter updates a held one in
|
||||
// place, so its messages stay; CreateDeclared makes one the bus lacks.
|
||||
type sourcing struct {
|
||||
Creator
|
||||
filters map[string]string // by stream, its source filter from EVENTS
|
||||
messages map[string]int
|
||||
made []string
|
||||
}
|
||||
|
||||
func (s *sourcing) SourceFilter(stream, source, filter string) (string, bool, error) {
|
||||
from, ok := s.filters[stream]
|
||||
if !ok || source != buscompose.EventsStream || from == filter {
|
||||
return from, false, nil
|
||||
}
|
||||
s.filters[stream] = filter
|
||||
return from, true, nil
|
||||
}
|
||||
|
||||
func (s *sourcing) CreateDeclared(cfg jetstream.StreamConfig) error {
|
||||
s.made = append(s.made, cfg.Name)
|
||||
s.filters[cfg.Name], s.messages[cfg.Name] = cfg.Sources[0].FilterSubject, 0
|
||||
return nil
|
||||
}
|
||||
|
||||
// H-A-145 (design 47 H §746): a held kept stream whose source filter the definition changed is brought to the definition's
|
||||
// filter in place, its messages kept and the change said with both filters; one at the definition's filter is untouched; one
|
||||
// the bus lacks is made.
|
||||
func TestAKeptStreamFollowsItsDefinitionsSourceFilter(t *testing.T) {
|
||||
const ended = "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.*.*"
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
held string // the held stream's source filter; "" for a stream the bus lacks
|
||||
made bool
|
||||
moved string
|
||||
filter string
|
||||
messages, keptMsgs int
|
||||
}{
|
||||
{name: "held at the definition's filter: untouched", held: ended, filter: ended, messages: 604, keptMsgs: 604},
|
||||
{name: "held at an earlier filter: brought to the definition's, messages kept", held: ended + ".*", filter: ended, messages: 604, keptMsgs: 604,
|
||||
moved: "SEAT_mesh-pipeline-runner_pipeline-ended: source filter " + ended + ".* brought to " + ended + ", its messages kept"},
|
||||
{name: "not held: made", made: true, filter: ended},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
k := buscompose.KeptEvent{Seat: "mesh-pipeline-runner", Event: "pipeline-ended", Stream: "SEAT_mesh-pipeline-runner_pipeline-ended", Filter: ended, MaxAgeDays: 90, MaxMiB: 192}
|
||||
c := &sourcing{filters: map[string]string{}, messages: map[string]int{}}
|
||||
held := map[string]Limits{}
|
||||
if row.held != "" {
|
||||
c.filters[k.Stream], c.messages[k.Stream] = row.held, row.messages
|
||||
held[k.Stream] = Limits{}
|
||||
}
|
||||
made, moved, notMade := createKept(c, &creations{}, []buscompose.KeptEvent{k}, held, false)
|
||||
if len(notMade) != 0 {
|
||||
t.Fatalf("not made: %v", notMade)
|
||||
}
|
||||
if (len(made) == 1) != row.made || len(c.made) != len(made) {
|
||||
t.Errorf("made %v (bus %v), want made %v", made, c.made, row.made)
|
||||
}
|
||||
if got := strings.Join(moved, "; "); got != row.moved {
|
||||
t.Errorf("moved %q, want %q", got, row.moved)
|
||||
}
|
||||
if c.filters[k.Stream] != row.filter || c.messages[k.Stream] != row.keptMsgs {
|
||||
t.Errorf("the stream holds filter %q and %d messages, want %q and %d", c.filters[k.Stream], c.messages[k.Stream], row.filter, row.keptMsgs)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -65,14 +65,24 @@ func keptOutOfHearing(kept []buscompose.KeptEvent, watches []buscompose.Seat, co
|
||||
}
|
||||
|
||||
// createKept makes each kept event's stream the bus lacks, from the shared events stream's messages of its subject, and
|
||||
// says what it made. A stream the bus holds is left as it is; one made at this seat list and gone since is made again only on
|
||||
// a hand assert (the same rule as a manifest's streams: made once per definition). held is what the bus held at this pass.
|
||||
func createKept(c Creator, done *creations, kept []buscompose.KeptEvent, held map[string]Limits, force bool) (made, notMade []string) {
|
||||
// says what it made. A stream the bus holds keeps its messages and is not made again; its source filter is brought to the
|
||||
// definition's in place when the definition changed the event's subject (design 47 H H-A-145, §746: the runner's
|
||||
// pipeline-ended moved from three parts to two and its kept stream took nothing in), said in moved with the held and the
|
||||
// composed filter. One made at this seat list and gone since is made again only on a hand assert (the same rule as a
|
||||
// manifest's streams: made once per definition). held is what the bus held at this pass.
|
||||
func createKept(c Creator, done *creations, kept []buscompose.KeptEvent, held map[string]Limits, force bool) (made, moved, notMade []string) {
|
||||
for _, k := range kept {
|
||||
cfg := k.StreamConfig()
|
||||
if _, ok := held[k.Stream]; ok {
|
||||
from, changed, err := c.SourceFilter(k.Stream, buscompose.EventsStream, k.Filter)
|
||||
switch {
|
||||
case err != nil:
|
||||
notMade = append(notMade, k.Stream+": its source filter is not brought to "+k.Filter+": "+err.Error()+" (tried again at the next pass)")
|
||||
case changed:
|
||||
moved = append(moved, k.Stream+": source filter "+from+" brought to "+k.Filter+", its messages kept")
|
||||
}
|
||||
continue
|
||||
}
|
||||
cfg := k.StreamConfig()
|
||||
digest := fmt.Sprintf("%v", cfg.Sources[0].FilterSubject) + fmt.Sprint(k.Last, k.MaxAgeDays, k.MaxMiB)
|
||||
if !force && done.madeAt(k.Stream) == digest {
|
||||
notMade = append(notMade, k.Stream+": made at this definition and gone since; grants_assert makes it again")
|
||||
@@ -87,7 +97,7 @@ func createKept(c Creator, done *creations, kept []buscompose.KeptEvent, held ma
|
||||
notMade = append(notMade, k.Stream+": "+err.Error()+" (tried again at the next event)")
|
||||
}
|
||||
}
|
||||
return made, notMade
|
||||
return made, moved, notMade
|
||||
}
|
||||
|
||||
// eventSubjectsOf is, by seat, each event's subject pattern (`running.<node>.<module>`) of the events whose subject carries
|
||||
|
||||
@@ -565,11 +565,21 @@ func (p *Part) Compose() Result {
|
||||
}
|
||||
forced := ask.Assert
|
||||
if p.Create != nil && heldErr == nil {
|
||||
made, notMade := createKept(p.Create, &p.made, in.Kept, held, forced != "")
|
||||
made, moved, notMade := createKept(p.Create, &p.made, in.Kept, held, forced != "")
|
||||
if len(made) > 0 {
|
||||
res.Asserted = append(res.Asserted, "the seats' kept events: created "+strings.Join(made, ", "))
|
||||
p.say(&res, EventAsserted, map[string]any{"module": Module, "created": made})
|
||||
}
|
||||
for _, m := range moved {
|
||||
// Said in the journal with both filters (H-A-145).
|
||||
res.Asserted = append(res.Asserted, "the seats' kept events: "+m)
|
||||
if p.Logf != nil {
|
||||
p.Logf("kept stream %s", m)
|
||||
}
|
||||
}
|
||||
if len(moved) > 0 {
|
||||
p.say(&res, EventAsserted, map[string]any{"module": Module, "source-filters": moved})
|
||||
}
|
||||
for _, why := range notMade {
|
||||
res.NotAsserted = append(res.NotAsserted, "kept event "+why)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user