mesh-grants: bring a held kept stream to its definition's source filter

The kept stream of the runner's pipeline-ended was made at G's
definition, sourced by a three-part subject; H1 moved the event to
<branch>.<source>, so the stream took nothing in since 08:13 and the
assigner, which hears a kept event from its kept stream, heard no end.
createKept left every held kept stream as it was. It now brings a held
one's source filter from the events stream to the definition's in place
(a stream update, its messages and other settings kept, never made
again) and says the held and the composed filter in the journal and the
pass; the same holds for pipeline-started and wait-held.

Builds: design 47 H H-A-145 (hq design/47-H1-answers-59 0787704b), §746
This commit is contained in:
2026-11-20 15:42:24 +01:00
parent abb7cfcf5d
commit 380d4781e6
9 changed files with 169 additions and 11 deletions
@@ -110,6 +110,34 @@ func (c creating) RaiseMaxBytes(stream string, to int64) (int64, bool, error) {
return from, true, nil
}
// SourceFilter brings the filter subject of a held stream's source from `source` to `filter` and changes nothing else
// (design 47 H H-A-145): it reads the stream's configuration and updates with the rest as it was, so its messages stay. A
// stream with no source from `source`, or one already at `filter`, is left as it is.
func (c creating) SourceFilter(stream, source, filter string) (string, bool, error) {
ctx, cancel := context.WithTimeout(c.ctx, requestWithin)
defer cancel()
s, err := c.js.Stream(ctx, stream)
if err != nil {
return "", false, err
}
cfg := s.CachedInfo().Config
i := slices.IndexFunc(cfg.Sources, func(src *jetstream.StreamSource) bool { return src != nil && src.Name == source })
if i < 0 || cfg.Sources[i].FilterSubject == filter {
if i < 0 {
return "", false, nil
}
return cfg.Sources[i].FilterSubject, false, nil
}
from := cfg.Sources[i].FilterSubject
src := *cfg.Sources[i]
src.FilterSubject = filter
cfg.Sources[i] = &src
if _, err := c.js.UpdateStream(ctx, cfg); err != nil {
return from, false, err
}
return from, true, nil
}
// AddFilters adds filter subjects to a held consumer and changes nothing else.
func (c creating) AddFilters(stream, name string, add []string) error {
ctx, cancel := context.WithTimeout(c.ctx, requestWithin)
@@ -611,6 +611,35 @@ func TestTheGrantsPartWhole(t *testing.T) {
if again, _ := stream.Info(ctx); !slices.Equal(again.Config.Subjects, []string{"a.>", "b.>"}) {
t.Errorf("subjects after adding a held one: %v", again.Config.Subjects)
}
// A kept stream's source filter is brought to its definition's in place: the stream is not made again, its settings
// stay, and an equal filter or a source of another stream changes nothing (design 47 H H-A-145, §746).
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "KEPTSRC", Subjects: []string{"ev.>"}}); err != nil {
t.Fatal(err)
}
kept, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "KEPT", MaxAge: time.Hour,
Sources: []*jetstream.StreamSource{{Name: "KEPTSRC", FilterSubject: "ev.ended.*.*.*"}}})
if err != nil {
t.Fatal(err)
}
keptBefore, _ := kept.Info(ctx)
for _, row := range []struct {
source, filter, from, holds string
changed bool
}{
{"KEPTSRC", "ev.ended.*.*", "ev.ended.*.*.*", "ev.ended.*.*", true},
{"KEPTSRC", "ev.ended.*.*", "ev.ended.*.*", "ev.ended.*.*", false},
{"OTHER", "ev.other", "", "ev.ended.*.*", false},
} {
from, changed, err := c.SourceFilter("KEPT", row.source, row.filter)
if err != nil {
t.Fatal(err)
}
now, _ := kept.Info(ctx)
if from != row.from || changed != row.changed || len(now.Config.Sources) != 1 || now.Config.Sources[0].FilterSubject != row.holds ||
now.Config.MaxAge != keptBefore.Config.MaxAge || !now.Created.Equal(keptBefore.Created) {
t.Errorf("source %s to %s: from %q changed %v, config %+v created %v", row.source, row.filter, from, changed, now.Config, now.Created)
}
}
// The named asks' purge takes one subject's messages and no other's, and a named worker's delete is said done also
// where the bus holds none (design 47 G §544).
for _, subj := range []string{"b.one", "b.two"} {
+7 -4
View File
@@ -202,6 +202,9 @@ func (b *fakeBus) AddSubjects(stream string, add []string) error {
return errors.New("stream not found")
}
// SourceFilter is the bus's update of a held stream's source filter: the fake holds no sources, so nothing changes.
func (b *fakeBus) SourceFilter(string, string, string) (string, bool, error) { return "", false, nil }
// RaiseMaxBytes is the bus's raise of a held stream's size cap: only to a larger cap, an unbounded stream and every other
// setting kept.
func (b *fakeBus) RaiseMaxBytes(stream string, to int64) (int64, bool, error) {
@@ -684,10 +687,10 @@ func TestGrants(t *testing.T) {
if n := strings.Count(string(src), "CreateStream"); (path == createOnly && n != 1) || (path != createOnly && n != 0) {
t.Errorf("%s holds CreateStream %d times: only create.go creates, once", path, n)
}
// The two updates of a stream: adding subjects to a held one (the operator, 2026-11-05) and raising its size cap
// (2026-11-09), both in create.go.
if n := strings.Count(string(src), "UpdateStream("); (path == createOnly && n != 2) || (path != createOnly && n != 0) {
t.Errorf("%s holds UpdateStream %d times: only create.go updates a stream, twice, to add subjects and to raise a size cap", path, n)
// The three updates of a stream: adding subjects to a held one (the operator, 2026-11-05), raising its size cap
// (2026-11-09) and bringing a kept stream's source filter to its definition's (design 47 H H-A-145), all in create.go.
if n := strings.Count(string(src), "UpdateStream("); (path == createOnly && n != 3) || (path != createOnly && n != 0) {
t.Errorf("%s holds UpdateStream %d times: only create.go updates a stream, three times, to add subjects, to raise a size cap and to bring a kept stream's source filter", path, n)
}
// The one object bucket: the part's own, of the user list sealed to the bus node, in sealed.go (ADR 0547 clause 7).
if path == sealedOnly {
@@ -1765,7 +1765,8 @@ const WatchmanModule = "mesh-delivery-watchman"
// 2026-10-26 08:54): its create, and its information to read back what it holds. Each by its own name: a name that
// is not one plain subject token is left out, since a wildcard there would be every stream. Its update is granted for the one
// acts the part does on a held stream: adding the subjects a manifest's new events need (grants.Creator.AddSubjects) and
// raising its size cap to the manifest's when larger (grants.Creator.RaiseMaxBytes; the operator, 2026-11-09); it never
// raising its size cap to the manifest's when larger (grants.Creator.RaiseMaxBytes; the operator, 2026-11-09), and bringing
// a kept stream's source filter to its definition's (grants.Creator.SourceFilter; design 47 H H-A-145); it never
// removes a subject, lowers a limit or recreates the stream, and never deletes or purges one.
func CreateStreamGrants(streams []string) []string {
var out []string
@@ -21,6 +21,9 @@ func (c *askCreator) AddSubjects(string, []string) error { return nil }
func (c *askCreator) RaiseMaxBytes(string, int64) (int64, bool, error) {
return 0, false, nil
}
func (c *askCreator) SourceFilter(string, string, string) (string, bool, error) {
return "", false, nil
}
// A seat whose queues came with G (mesh-builds) gets its work-queue ask stream from the seat list, as the controller made one for
// every seat before G; a seat that accepts nothing gets none, and a stream the bus holds is never made again (lead-47, 2026-11-15).
@@ -69,6 +69,10 @@ type Creator interface {
// changes nothing else (the operator, 2026-11-09 09:07: "raise only"). It answers the cap it held and whether it
// raised; a cap that is larger or equal, or an unbounded stream, is left as it is. It never lowers any limit.
RaiseMaxBytes(stream string, to int64) (from int64, raised bool, err error)
// SourceFilter brings the filter subject of a held stream's source from the stream `source` to `filter`, keeping its
// messages and every other setting, the stream not made again (design 47 H H-A-145). It answers the filter it held and
// whether it changed; a stream with no source from `source` is left as it is.
SourceFilter(stream, source, filter string) (from string, changed bool, err error)
}
// Raise is one held stream's size cap raised to the manifest's, said in the event and the journal.
@@ -1,8 +1,11 @@
package grants
import (
"strings"
"testing"
"github.com/nats-io/nats.go/jetstream"
"mesh-grants/internal/buscompose"
)
@@ -39,3 +42,70 @@ func TestTheGrantsAccountMayCreateEachKeptEventStream(t *testing.T) {
})
}
}
// sourcing is a Creator holding kept streams by their source filter and message count: SourceFilter updates a held one in
// place, so its messages stay; CreateDeclared makes one the bus lacks.
type sourcing struct {
Creator
filters map[string]string // by stream, its source filter from EVENTS
messages map[string]int
made []string
}
func (s *sourcing) SourceFilter(stream, source, filter string) (string, bool, error) {
from, ok := s.filters[stream]
if !ok || source != buscompose.EventsStream || from == filter {
return from, false, nil
}
s.filters[stream] = filter
return from, true, nil
}
func (s *sourcing) CreateDeclared(cfg jetstream.StreamConfig) error {
s.made = append(s.made, cfg.Name)
s.filters[cfg.Name], s.messages[cfg.Name] = cfg.Sources[0].FilterSubject, 0
return nil
}
// H-A-145 (design 47 H §746): a held kept stream whose source filter the definition changed is brought to the definition's
// filter in place, its messages kept and the change said with both filters; one at the definition's filter is untouched; one
// the bus lacks is made.
func TestAKeptStreamFollowsItsDefinitionsSourceFilter(t *testing.T) {
const ended = "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.*.*"
for _, row := range []struct {
name string
held string // the held stream's source filter; "" for a stream the bus lacks
made bool
moved string
filter string
messages, keptMsgs int
}{
{name: "held at the definition's filter: untouched", held: ended, filter: ended, messages: 604, keptMsgs: 604},
{name: "held at an earlier filter: brought to the definition's, messages kept", held: ended + ".*", filter: ended, messages: 604, keptMsgs: 604,
moved: "SEAT_mesh-pipeline-runner_pipeline-ended: source filter " + ended + ".* brought to " + ended + ", its messages kept"},
{name: "not held: made", made: true, filter: ended},
} {
t.Run(row.name, func(t *testing.T) {
k := buscompose.KeptEvent{Seat: "mesh-pipeline-runner", Event: "pipeline-ended", Stream: "SEAT_mesh-pipeline-runner_pipeline-ended", Filter: ended, MaxAgeDays: 90, MaxMiB: 192}
c := &sourcing{filters: map[string]string{}, messages: map[string]int{}}
held := map[string]Limits{}
if row.held != "" {
c.filters[k.Stream], c.messages[k.Stream] = row.held, row.messages
held[k.Stream] = Limits{}
}
made, moved, notMade := createKept(c, &creations{}, []buscompose.KeptEvent{k}, held, false)
if len(notMade) != 0 {
t.Fatalf("not made: %v", notMade)
}
if (len(made) == 1) != row.made || len(c.made) != len(made) {
t.Errorf("made %v (bus %v), want made %v", made, c.made, row.made)
}
if got := strings.Join(moved, "; "); got != row.moved {
t.Errorf("moved %q, want %q", got, row.moved)
}
if c.filters[k.Stream] != row.filter || c.messages[k.Stream] != row.keptMsgs {
t.Errorf("the stream holds filter %q and %d messages, want %q and %d", c.filters[k.Stream], c.messages[k.Stream], row.filter, row.keptMsgs)
}
})
}
}
+15 -5
View File
@@ -65,14 +65,24 @@ func keptOutOfHearing(kept []buscompose.KeptEvent, watches []buscompose.Seat, co
}
// createKept makes each kept event's stream the bus lacks, from the shared events stream's messages of its subject, and
// says what it made. A stream the bus holds is left as it is; one made at this seat list and gone since is made again only on
// a hand assert (the same rule as a manifest's streams: made once per definition). held is what the bus held at this pass.
func createKept(c Creator, done *creations, kept []buscompose.KeptEvent, held map[string]Limits, force bool) (made, notMade []string) {
// says what it made. A stream the bus holds keeps its messages and is not made again; its source filter is brought to the
// definition's in place when the definition changed the event's subject (design 47 H H-A-145, §746: the runner's
// pipeline-ended moved from three parts to two and its kept stream took nothing in), said in moved with the held and the
// composed filter. One made at this seat list and gone since is made again only on a hand assert (the same rule as a
// manifest's streams: made once per definition). held is what the bus held at this pass.
func createKept(c Creator, done *creations, kept []buscompose.KeptEvent, held map[string]Limits, force bool) (made, moved, notMade []string) {
for _, k := range kept {
cfg := k.StreamConfig()
if _, ok := held[k.Stream]; ok {
from, changed, err := c.SourceFilter(k.Stream, buscompose.EventsStream, k.Filter)
switch {
case err != nil:
notMade = append(notMade, k.Stream+": its source filter is not brought to "+k.Filter+": "+err.Error()+" (tried again at the next pass)")
case changed:
moved = append(moved, k.Stream+": source filter "+from+" brought to "+k.Filter+", its messages kept")
}
continue
}
cfg := k.StreamConfig()
digest := fmt.Sprintf("%v", cfg.Sources[0].FilterSubject) + fmt.Sprint(k.Last, k.MaxAgeDays, k.MaxMiB)
if !force && done.madeAt(k.Stream) == digest {
notMade = append(notMade, k.Stream+": made at this definition and gone since; grants_assert makes it again")
@@ -87,7 +97,7 @@ func createKept(c Creator, done *creations, kept []buscompose.KeptEvent, held ma
notMade = append(notMade, k.Stream+": "+err.Error()+" (tried again at the next event)")
}
}
return made, notMade
return made, moved, notMade
}
// eventSubjectsOf is, by seat, each event's subject pattern (`running.<node>.<module>`) of the events whose subject carries
+11 -1
View File
@@ -565,11 +565,21 @@ func (p *Part) Compose() Result {
}
forced := ask.Assert
if p.Create != nil && heldErr == nil {
made, notMade := createKept(p.Create, &p.made, in.Kept, held, forced != "")
made, moved, notMade := createKept(p.Create, &p.made, in.Kept, held, forced != "")
if len(made) > 0 {
res.Asserted = append(res.Asserted, "the seats' kept events: created "+strings.Join(made, ", "))
p.say(&res, EventAsserted, map[string]any{"module": Module, "created": made})
}
for _, m := range moved {
// Said in the journal with both filters (H-A-145).
res.Asserted = append(res.Asserted, "the seats' kept events: "+m)
if p.Logf != nil {
p.Logf("kept stream %s", m)
}
}
if len(moved) > 0 {
p.say(&res, EventAsserted, map[string]any{"module": Module, "source-filters": moved})
}
for _, why := range notMade {
res.NotAsserted = append(res.NotAsserted, "kept event "+why)
}