From 0cf419619e9cb14e6f685e5bcbd29f1f7ac85dba Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 12:10:46 +0100 Subject: [PATCH 1/3] indexer: make a watched branch's source-moved durable when the bus holds none MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The grants part composes no per-branch durable on the source-moved stream, so the indexer never heard a source revision after its start. Its bind now makes the durable itself, as a module's seat-event consumer is made (SeatEventConsumerConfig, named __, filtered to the branch, one message at a time), and binds an existing one as it stands. Builds: design 47 H §743, H-A-134 Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy --- .../cmd/mesh-delivery-indexer/sourcemoved.go | 22 ++++- .../mesh-delivery-indexer/sourcemoved_test.go | 80 +++++++++++++++++++ 2 files changed, 101 insertions(+), 1 deletion(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go index 51cdc875..71587421 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go @@ -79,10 +79,19 @@ func sourceVia(call toolCaller) func(context.Context, string) (meshdelivery.Sour } } -// bindBranch binds the indexer's durable of one watched branch on the stream of source-moved through mesh-sdk. +// bindBranch binds the indexer's durable of one watched branch on the stream of source-moved through mesh-sdk. One the bus does +// not hold the indexer makes itself, as its own kept consumer, and then binds (design 47 H §743, H-A-134: the grants part +// composes none until H4b); one the bus holds is bound as it stands, never made again. func bindBranch(js jetstream.JetStream, cred process.Account, logf func(string, ...any)) func(context.Context, string, func(jetstream.Msg)) (func(), error) { return func(ctx context.Context, branch string, each func(jetstream.Msg)) (func(), error) { cons, err := cred.NamedEventConsumer(ctx, js, meshdelivery.StreamSourceMoved, keys.EscapeName(branch)) + if errors.Is(err, jetstream.ErrConsumerNotFound) { + cfg := branchConsumerConfig(cred, branch) + if cons, err = js.CreateConsumer(ctx, meshdelivery.StreamSourceMoved, cfg); err != nil { + return nil, fmt.Errorf("the consumer %s on %s is not there and is not made: %w", cfg.Durable, meshdelivery.StreamSourceMoved, err) + } + logf("source-moved: the consumer %s on %s was not there: the indexer made it (H-A-134)", cfg.Durable, meshdelivery.StreamSourceMoved) + } if err != nil { return nil, err } @@ -94,6 +103,17 @@ func bindBranch(js jetstream.JetStream, cred process.Account, logf func(string, } } +// branchConsumerConfig is the indexer's durable of one watched branch as the mesh gives a module's seat-event consumer +// (mesh-sdk's SeatEventConsumerConfig), named __ as NamedEventConsumer binds it, filtered to the +// branch's source-moved and delivering one message at a time (design 47 H §743, H-A-90, H-A-134). +func branchConsumerConfig(cred process.Account, branch string) jetstream.ConsumerConfig { + esc := keys.EscapeName(branch) + cfg := cred.SeatEventConsumerConfig([]string{meshdelivery.SubjectSourceMoved(esc, "*")}, false) + cfg.Durable = cred.EventConsumerName() + "_" + esc + cfg.MaxAckPending = 1 + return cfg +} + // refresh binds the consumer of every watched branch not bound yet: at the start and on each registration-changed. A consumer // not there yet is said and bound at the next registration-changed or start. The start's refresh reads every watched branch's // source revision, a later one each branch whose consumer could not be bound (H-A-129), after the binds. diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved_test.go index fcd97d79..9b9fcfa0 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved_test.go @@ -12,6 +12,8 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" + "git.novox.be/novox/mesh-sdk/go/keys" + "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-indexer/internal/index" ) @@ -152,3 +154,81 @@ func TestAStartWithNoConsumerTakesTheGatesSourceRevisionIn(t *testing.T) { t.Errorf("asked the gate for %v and said %v; want main asked at each and taken in once", asked, taken) } } + +// fakeBranchJS is the bus as bindBranch uses it: the source-moved stream's consumers by name, and the consumers made. +type fakeBranchJS struct { + jetstream.JetStream + held map[string]*fakeBranchConsumer + made []jetstream.ConsumerConfig +} + +func (f *fakeBranchJS) Consumer(_ context.Context, stream, name string) (jetstream.Consumer, error) { + if c, ok := f.held[stream+"/"+name]; ok { + return c, nil + } + return nil, jetstream.ErrConsumerNotFound +} + +func (f *fakeBranchJS) CreateConsumer(_ context.Context, stream string, cfg jetstream.ConsumerConfig) (jetstream.Consumer, error) { + f.made = append(f.made, cfg) + c := &fakeBranchConsumer{} + f.held[stream+"/"+cfg.Durable] = c + return c, nil +} + +// fakeBranchConsumer hands one source-moved to the hearing as soon as it is consumed. +type fakeBranchConsumer struct { + jetstream.Consumer + heard int +} + +type fakeConsumeContext struct{ jetstream.ConsumeContext } + +func (fakeConsumeContext) Stop() {} + +func (c *fakeBranchConsumer) Consume(handle jetstream.MessageHandler, _ ...jetstream.PullConsumeOpt) (jetstream.ConsumeContext, error) { + c.heard++ + handle(msgOf{f: &fakeMsg{subject: "mesh.seat.mesh-delivery.event.source-moved.main.2"}}) + return fakeConsumeContext{}, nil +} + +// Design 47 H §743, H-A-134: a branch's durable the bus does not hold the indexer makes, as a module's seat-event consumer is +// made (SeatEventConsumerConfig), named __, filtered to the branch, one message at a time, and then +// hears on it; one the bus holds is bound, never made again. +func TestABranchDurableNotThereIsMadeAndHeard(t *testing.T) { + cred := process.Account{Node: "n1", Module: "mesh-delivery-indexer"} + for _, tc := range []struct { + name string + held bool + }{{"not there: made", false}, {"there: bound", true}} { + t.Run(tc.name, func(t *testing.T) { + js := &fakeBranchJS{held: map[string]*fakeBranchConsumer{}} + name := "n1_mesh-delivery-indexer_" + keys.EscapeName("main") + var existing *fakeBranchConsumer + if tc.held { + existing = &fakeBranchConsumer{} + js.held[meshdelivery.StreamSourceMoved+"/"+name] = existing + } + var heard []string + stop, err := bindBranch(js, cred, func(string, ...any) {})(context.Background(), "main", func(m jetstream.Msg) { heard = append(heard, m.Subject()) }) + if err != nil { + t.Fatal(err) + } + stop() + if len(heard) != 1 { + t.Fatalf("heard %v, want the one source-moved the consumer hands", heard) + } + if tc.held { + if len(js.made) != 0 || existing.heard != 1 { + t.Fatalf("made %v and heard the held one %d times; want it bound, not made again", js.made, existing.heard) + } + return + } + want := cred.SeatEventConsumerConfig([]string{meshdelivery.SubjectSourceMoved(keys.EscapeName("main"), "*")}, false) + want.Durable, want.MaxAckPending = name, 1 + if len(js.made) != 1 || !reflect.DeepEqual(js.made[0], want) { + t.Fatalf("made %+v, want once %+v", js.made, want) + } + }) + } +} From f95a1649414b6c27bd8830175ff6228f6c7418ad Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 12:12:23 +0100 Subject: [PATCH 2/3] indexer: drop the earlier model's trunk and branch revisions at a take-in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit catalog-revisions kept trunk. and branch.. objects beside the branch-keyed revisions, the newest trunk one marked current, so a reader heard revision 528 while its picture held 1. A branch's take-in now drops every such object no record pins, says the count in the journal, and the revisions tool marks each branch's newest revision current. Builds: design 47 H §743, H-A-135 Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy --- .../internal/index/catalog_revisions.go | 5 ++ .../internal/index/pins.go | 83 +++++++++++++++---- .../internal/index/revision_tools.go | 11 ++- .../internal/index/takein_source_test.go | 34 ++++++++ 4 files changed, 111 insertions(+), 22 deletions(-) diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index e584f836..d340e193 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -337,6 +337,11 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int } else if len(removed) > 0 { ix.logf("pins: %d revisions removed (%s), no record pins them and none is current", len(removed), strings.Join(removed, ", ")) } + if dropped, err := ix.dropEarlierFormsLocked(ctx); err != nil { + ix.logf("catalog graph: the revisions of the earlier forms (trunk.*, branch.*) are not dropped: %v", err) + } else if len(dropped) > 0 { + ix.logf("catalog graph: %d revisions of the earlier forms dropped (%s), no record pins them; each branch's newest revision is its current one", len(dropped), strings.Join(dropped, ", ")) + } if _, err := ix.removeUnlisted(ctx); err != nil { ix.logf("catalog graph: the entries nothing lists or names any more are not removed: %v", err) // at the first revision after none does (§675) } diff --git a/modules/mesh-delivery-indexer/internal/index/pins.go b/modules/mesh-delivery-indexer/internal/index/pins.go index 3140815d..439c97c2 100644 --- a/modules/mesh-delivery-indexer/internal/index/pins.go +++ b/modules/mesh-delivery-indexer/internal/index/pins.go @@ -253,25 +253,10 @@ func (ix *Indexer) judgePinsLocked(ctx context.Context) ([]string, error) { return nil, err } current := CurrentRevisions(names) - pinned := map[string]bool{} - pinKeys, err := ix.Pins.Keys(ctx) + pinKeys, pinned, err := ix.pinnedObjects(ctx) if err != nil { return nil, err } - for _, k := range pinKeys { - object, ok := pinObject(k) - if !ok { - continue - } - v, _, found, err := ix.Pins.Get(ctx, k) - if err != nil { - return nil, err - } - var pv PinValue - if found && json.Unmarshal(v, &pv) == nil && len(pv.Records) > 0 { - pinned[object] = true - } - } var removed []string for _, n := range names { if _, _, ok := RevisionGroup(n); !ok || pinned[n] { @@ -301,6 +286,72 @@ func (ix *Indexer) judgePinsLocked(ctx context.Context) ([]string, error) { return removed, nil } +// pinnedObjects are the state pins' keys and the revision objects a record pins. +func (ix *Indexer) pinnedObjects(ctx context.Context) ([]string, map[string]bool, error) { + pinned := map[string]bool{} + pinKeys, err := ix.Pins.Keys(ctx) + if err != nil { + return nil, nil, err + } + for _, k := range pinKeys { + object, ok := pinObject(k) + if !ok { + continue + } + v, _, found, err := ix.Pins.Get(ctx, k) + if err != nil { + return nil, nil, err + } + var pv PinValue + if found && json.Unmarshal(v, &pv) == nil && len(pv.Records) > 0 { + pinned[object] = true + } + } + return pinKeys, pinned, nil +} + +// earlierForm says whether a revision object is of the earlier model's forms, trunk. or branch.., beside the +// branch-keyed . of design 47 H §743 (H-A-135). +func earlierForm(name string) bool { + if _, _, ok := RevisionGroup(name); !ok { + return false + } + return strings.HasPrefix(name, "trunk.") || strings.HasPrefix(name, "branch.") +} + +// dropEarlierFormsLocked drops every revision object of the earlier forms (earlierForm) no record pins, at a branch's take-in +// (design 47 H §743, H-A-135): the branch's newest revision is its current one, and an earlier trunk revision kept beside it +// would be the newest trunk revision a reader hears. It drops none while the pins do not count yet; the caller holds keepMu. +func (ix *Indexer) dropEarlierFormsLocked(ctx context.Context) ([]string, error) { + if ix.PinsPending != nil { + if pending, err := ix.PinsPending(ctx); err != nil || pending != 0 { + return nil, err + } + } + names, err := ix.Revisions.Names(ctx) + if err != nil { + return nil, err + } + pinned := map[string]bool{} + if ix.Pins != nil { + if _, pinned, err = ix.pinnedObjects(ctx); err != nil { + return nil, err + } + } + var dropped []string + for _, n := range names { + if !earlierForm(n) || pinned[n] { + continue + } + if err := ix.Revisions.Delete(ctx, n); err != nil { + return dropped, fmt.Errorf("%s: %w", n, err) + } + dropped = append(dropped, n) + } + sort.Strings(dropped) + return dropped, nil +} + // PinDrop drops one pin of the state pins by hand, by its key . (indexer_pin_drop, design 47 H §765, H-A-57): // every record's, for a pin that names no running pipeline; the finding pin-missing of the key is cleared, and the removal // rule then applies. diff --git a/modules/mesh-delivery-indexer/internal/index/revision_tools.go b/modules/mesh-delivery-indexer/internal/index/revision_tools.go index 24ba847e..279e41e8 100644 --- a/modules/mesh-delivery-indexer/internal/index/revision_tools.go +++ b/modules/mesh-delivery-indexer/internal/index/revision_tools.go @@ -61,7 +61,8 @@ type RevisionSummary struct { Heads string `json:"heads,omitempty"` At *time.Time `json:"at,omitempty"` Entries int `json:"entries"` - // Current says it is the trunk's newest revision, which is kept whatever pins it. + // Current says it is the newest revision of its watched branch (CurrentRevisions, §579; H §743, H-A-135: no longer the + // newest trunk-kind revision over every branch, and never one of the earlier forms), which is kept whatever pins it. Current bool `json:"current,omitempty"` } @@ -84,7 +85,6 @@ func ListRevisions(ctx context.Context, o Objects, candidates bool) (RevisionsAn return out, fmt.Errorf("the objects %s cannot be listed: %w", CatalogRevisionsObjects, err) } listed := map[string]bool{} - var newest int64 entries := []string{} for _, n := range names { if strings.HasPrefix(n, "entry.") { @@ -103,17 +103,16 @@ func ListRevisions(ctx context.Context, o Objects, candidates bool) (RevisionsAn for _, ref := range rev.Entries { listed[EntryObject(ref.Digest)] = true } - if rev.Kind == "trunk" && rev.Revision > newest { - newest = rev.Revision - } if candidates && rev.Kind != "candidate" { continue } out.Revisions = append(out.Revisions, RevisionSummary{Name: n, Kind: rev.Kind, Revision: rev.Revision, Source: rev.Source, Group: rev.Group, Heads: rev.Heads, At: rev.At, Entries: len(rev.Entries)}) } + current := CurrentRevisions(names) for i := range out.Revisions { - out.Revisions[i].Current = out.Revisions[i].Kind == "trunk" && out.Revisions[i].Revision == newest + _, cur := current[out.Revisions[i].Name] + out.Revisions[i].Current = cur && out.Revisions[i].Kind == "trunk" && !earlierForm(out.Revisions[i].Name) } slices.SortFunc(out.Revisions, func(a, b RevisionSummary) int { return strings.Compare(a.Name, b.Name) }) out.Entries = len(entries) diff --git a/modules/mesh-delivery-indexer/internal/index/takein_source_test.go b/modules/mesh-delivery-indexer/internal/index/takein_source_test.go index 7545a99f..d36933e0 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein_source_test.go +++ b/modules/mesh-delivery-indexer/internal/index/takein_source_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "reflect" "testing" @@ -121,6 +122,39 @@ func TestAFirstSourceRevisionIsTakenInWithNoMover(t *testing.T) { } } +// Design 47 H §743, H-A-135: at a branch's take-in the earlier model's revisions (trunk., branch..) no record +// pins are dropped, one a record pins stays, and the branch's newest revision is the current one, no longer the newest trunk. +func TestATakeInDropsTheEarlierFormsNoRecordPins(t *testing.T) { + ctx := context.Background() + ix, objects, _, f := takeInIndexer(t) + for _, n := range []string{"trunk.527", "trunk.528", "branch.master.1"} { + _, rev, _ := RevisionGroup(n) + raw, _ := json.Marshal(meshcatalog.CatalogRevision{Revision: rev, Kind: "trunk", Source: 1000 + rev}) + if err := objects.Put(ctx, n, raw); err != nil { + t.Fatal(err) + } + } + if err := ix.pin(ctx, "trunk.527", "pipeline.old"); err != nil { + t.Fatal(err) + } + start := meshdelivery.SourceMovedEvent{V: 1, Branch: "main", Source: 1, Call: "seed.operator.20261120", Start: true, + Heads: []meshdelivery.SourceMovedEventHeadsItem{{Repository: f.Repository, Commit: f.Merge}}} + if _, err := ix.TakeInSource(ctx, start); err != nil { + t.Fatal(err) + } + a, err := ListRevisions(ctx, objects, false) + if err != nil { + t.Fatal(err) + } + var got []string + for _, r := range a.Revisions { + got = append(got, fmt.Sprintf("%s current=%t", r.Name, r.Current)) + } + if want := []string{"main.1 current=true", "trunk.527 current=false"}; !reflect.DeepEqual(got, want) { + t.Errorf("kept %v, want %v: the unpinned earlier forms dropped, the pinned one kept, main.1 current", got, want) + } +} + // Design 47 H §769: the take-in of a source revision moving the catalogue repository reads `predefined` from that commit's // trees, and one changing a predefined key rewrites it. func TestATakeInOfTheCatalogsCommitRewritesPredefined(t *testing.T) { From e54bf7f34639fe6fdcf5ca781bbaf6a968cc1cb6 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 12:19:50 +0100 Subject: [PATCH 3/3] indexer: a branch's durable it makes waits 30 s, delivers 5 times, starts from now MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As the grants part gives a module's consumer on a seat event's stream kept by age: without a start from now a made durable would replay every past source-moved of the branch past the start's guard; the current revision is read at the start already. Builds: design 47 H §743, H-A-134 (hq 8d30cc42), H-A-129, H-A-130 Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy --- .../cmd/mesh-delivery-indexer/sourcemoved.go | 14 +++++++++++++- .../cmd/mesh-delivery-indexer/sourcemoved_test.go | 2 ++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go index 71587421..cde7d759 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go @@ -7,6 +7,7 @@ import ( "fmt" "slices" "sync" + "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" @@ -105,15 +106,26 @@ func bindBranch(js jetstream.JetStream, cred process.Account, logf func(string, // branchConsumerConfig is the indexer's durable of one watched branch as the mesh gives a module's seat-event consumer // (mesh-sdk's SeatEventConsumerConfig), named __ as NamedEventConsumer binds it, filtered to the -// branch's source-moved and delivering one message at a time (design 47 H §743, H-A-90, H-A-134). +// branch's source-moved and delivering one message at a time, with the ack wait of 30 s, the at most 5 deliveries and the +// start from now the grants part gives a module's consumer on a seat event's stream kept by age (mesh-grants buscompose +// heardConsumer): the branch's current source revision is read at the start through the gate's source (H-A-129, H-A-130), so +// its past source-moved are never taken in again (design 47 H §743, H-A-90, H-A-134). func branchConsumerConfig(cred process.Account, branch string) jetstream.ConsumerConfig { esc := keys.EscapeName(branch) cfg := cred.SeatEventConsumerConfig([]string{meshdelivery.SubjectSourceMoved(esc, "*")}, false) cfg.Durable = cred.EventConsumerName() + "_" + esc cfg.MaxAckPending = 1 + cfg.AckWait, cfg.MaxDeliver = branchAckWait, branchMaxDeliver + cfg.DeliverPolicy = jetstream.DeliverNewPolicy return cfg } +// The ack wait and the deliveries of a branch's durable, as the grants part gives a module's consumers (H-A-134, H-A-133). +const ( + branchAckWait = 30 * time.Second + branchMaxDeliver = 5 +) + // refresh binds the consumer of every watched branch not bound yet: at the start and on each registration-changed. A consumer // not there yet is said and bound at the next registration-changed or start. The start's refresh reads every watched branch's // source revision, a later one each branch whose consumer could not be bound (H-A-129), after the binds. diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved_test.go index 9b9fcfa0..9b3feb36 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved_test.go @@ -8,6 +8,7 @@ import ( "slices" "strings" "testing" + "time" "github.com/nats-io/nats.go/jetstream" @@ -226,6 +227,7 @@ func TestABranchDurableNotThereIsMadeAndHeard(t *testing.T) { } want := cred.SeatEventConsumerConfig([]string{meshdelivery.SubjectSourceMoved(keys.EscapeName("main"), "*")}, false) want.Durable, want.MaxAckPending = name, 1 + want.AckWait, want.MaxDeliver, want.DeliverPolicy = 30*time.Second, 5, jetstream.DeliverNewPolicy if len(js.made) != 1 || !reflect.DeepEqual(js.made[0], want) { t.Fatalf("made %+v, want once %+v", js.made, want) }