From cc5f803f8797e391c109d1961091ae6aa7b6d8d4 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 01:12:04 +0100 Subject: [PATCH] I5: the registrations are read in one call, filtered and paged MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The verb registrations and the new tool indexer_registrations filter by module and repository while they read, at most 768 KB a page, capped with the cursor next: the tools bundle reads the state from the bus's cursor (mesh/state.page), a store that pages none by its keys in order; total is said when one page holds the whole read. Builds: design 47 H2 §806, §811, §779 --- .../cmd/mesh-delivery-indexer-tools/main.go | 5 ++ .../internal/index/registrations.go | 75 ++++++++++++++--- .../internal/index/registrations_tools.go | 50 ++++++----- .../index/registrations_tools_test.go | 84 +++++++++++++++++++ .../mesh-delivery-indexer/manifest_test.go | 4 +- modules/mesh-delivery-indexer/module.json | 3 +- 6 files changed, 187 insertions(+), 34 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index d1bf3dcac..6f7f219d9 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -156,6 +156,11 @@ func (k keptRead) Get(_ context.Context, key string) ([]byte, uint64, bool, erro func (k keptRead) Keys(context.Context) ([]string, error) { return k.s.Keys() } +// Page reads the state a page at a time from the bus's cursor (design 47 H2 §779): the registrations are read so (§806). +func (k keptRead) Page(cursor string, maxBytes int, budget time.Duration) (stdio.StatePage, error) { + return k.s.Page(cursor, maxBytes, budget) +} + func (keptRead) Create(context.Context, string, []byte) error { return errReadOnly } func (keptRead) Update(context.Context, string, []byte, uint64) error { return errReadOnly } diff --git a/modules/mesh-delivery-indexer/internal/index/registrations.go b/modules/mesh-delivery-indexer/internal/index/registrations.go index 519515998..9b0205979 100644 --- a/modules/mesh-delivery-indexer/internal/index/registrations.go +++ b/modules/mesh-delivery-indexer/internal/index/registrations.go @@ -16,7 +16,10 @@ import ( oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" gitseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" + stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/keys" + "git.novox.be/novox/mesh-sdk/go/views" ) // A module's registration, and the trusted change of one (design 47 G §2 to §4, §10, §342, §343, §667): the indexer keeps each @@ -169,15 +172,34 @@ func (r *Registry) Registration(ctx context.Context, module string) (meshcatalog return reg, true, nil } -// Registrations is the verb registrations (§4): every registration, or one module's, with its history. +// RegistrationsTool is the indexer's tool of the registrations (design 47 H2 §806): the verb registrations' answer, by hand. +const RegistrationsTool = "indexer_registrations" + +// StatePager reads a state a page at a time, in the order the bus wrote it, after an opaque cursor (mesh-sdk +// KeptState.Page, design 47 H2 §779). +type StatePager interface { + Page(cursor string, maxBytes int, budget time.Duration) (stdio.StatePage, error) +} + +// keyCursor is the cursor of a store read by its keys (a store that pages none): after this key, in key order. +const keyCursor = "key:" + +// RegistrationsAnswer is the verb registrations and the tool indexer_registrations (§4; design 47 H2 §806, §811): every +// registration, or one module's with its history, filtered by module and repository while it reads, at most views.PageBytes +// a page, capped with the cursor next when more is held; a store that pages (StatePager) is read from the bus's cursor, +// any other by its keys in order. total is how many the filters hold, said when one page holds the whole read. func (r *Registry) RegistrationsAnswer(ctx context.Context, in meshcatalog.RegistrationsInput) (meshcatalog.RegistrationsAnswer, error) { out := meshcatalog.RegistrationsAnswer{Registrations: []meshcatalog.Registration{}} + repository := trunk.RepositoryName(strings.TrimSpace(in.Repository)) + keep := func(reg meshcatalog.Registration) bool { + return (in.Module == "" || reg.Module == in.Module) && (repository == "" || trunk.SameRepository(reg.Origin, repository)) + } if in.Module != "" { reg, found, err := r.Registration(ctx, in.Module) if err != nil { return out, err } - if found { + if found && keep(reg) { out.Registrations = append(out.Registrations, reg) } if in.History && r.HistoryOf != nil { @@ -185,22 +207,53 @@ func (r *Registry) RegistrationsAnswer(ctx context.Context, in meshcatalog.Regis return out, fmt.Errorf("the history of %s cannot be read: %w", in.Module, err) } } + out.Total = int64(len(out.Registrations)) return out, nil } - all, err := r.Registrations.Keys(ctx) - if err != nil { - return out, err + match := func(value []byte) (meshcatalog.Registration, bool) { + var reg meshcatalog.Registration + return reg, json.Unmarshal(value, ®) == nil && keep(reg) } - slices.Sort(all) - for _, k := range all { - v, _, found, err := r.Registrations.Get(ctx, k) + if p, ok := r.Registrations.(StatePager); ok { + page, err := p.Page(in.Next, views.PageBytes, views.PageBudget) if err != nil { return out, err } - var reg meshcatalog.Registration - if found && json.Unmarshal(v, ®) == nil { - out.Registrations = append(out.Registrations, reg) + // A page holds at most views.PageBytes, so what the filters keep of it always fits one answer. + for _, e := range page.Entries { + if reg, ok := match(e.Value); ok { + out.Registrations = append(out.Registrations, reg) + } } + out.Next, out.Capped = page.Next, page.Next != "" + } else { + all, err := r.Registrations.Keys(ctx) + if err != nil { + return out, err + } + slices.Sort(all) + after, size := strings.TrimPrefix(in.Next, keyCursor), 0 + for _, k := range all { + if in.Next != "" && k <= after { + continue + } + v, _, found, err := r.Registrations.Get(ctx, k) + if err != nil { + return out, err + } + reg, ok := match(v) + if !found || !ok { + continue + } + if size += len(k) + len(v); size > views.PageBytes && len(out.Registrations) > 0 { + out.Capped, out.Next = true, keyCursor+after + return out, nil + } + out.Registrations, after = append(out.Registrations, reg), k + } + } + if in.Next == "" && !out.Capped { + out.Total = int64(len(out.Registrations)) } return out, nil } diff --git a/modules/mesh-delivery-indexer/internal/index/registrations_tools.go b/modules/mesh-delivery-indexer/internal/index/registrations_tools.go index 9baa5cf5b..0a61d5375 100644 --- a/modules/mesh-delivery-indexer/internal/index/registrations_tools.go +++ b/modules/mesh-delivery-indexer/internal/index/registrations_tools.go @@ -43,27 +43,37 @@ func RegistrationTools(registrations, proposals Store, ask func(key string, valu // in, never queued behind a running take-in (design 47 G A-370). Only a module's history is read from the stream the // indexer keeps, so that is still asked of the indexer. viaIndexer := act(ActRegistrations, true) + read := func(args map[string]any) (any, error) { + var in meshcatalog.RegistrationsInput + clean := map[string]any{} + for k, v := range args { + if k != "caller" && !strings.HasPrefix(k, "_") { + clean[k] = v + } + } + raw, err := json.Marshal(clean) + if err == nil { + err = json.Unmarshal(raw, &in) + } + if err != nil { + return nil, err + } + if in.History { + return viaIndexer(args) + } + return (&Registry{Registrations: registrations}).RegistrationsAnswer(context.Background(), in) + } return []stdio.Tool{ - {Name: meshcatalog.Seat + "." + meshcatalog.VerbRegistrations, Run: func(args map[string]any) (any, error) { - var in meshcatalog.RegistrationsInput - clean := map[string]any{} - for k, v := range args { - if k != "caller" && !strings.HasPrefix(k, "_") { - clean[k] = v - } - } - raw, err := json.Marshal(clean) - if err == nil { - err = json.Unmarshal(raw, &in) - } - if err != nil { - return nil, err - } - if in.History { - return viaIndexer(args) - } - return (&Registry{Registrations: registrations}).RegistrationsAnswer(context.Background(), in) - }}, + {Name: meshcatalog.Seat + "." + meshcatalog.VerbRegistrations, Run: read}, + {Name: RegistrationsTool, Description: "The registrations (design 47 H2 §806), as the seat mesh-catalog's verb registrations answers them: " + + "every module's and every source-only one, filtered by module or repository (owner/name) while it reads, at most 768 KB a page " + + "(capped), the rest read from the cursor next; one module's with every version (history).", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "one module; absent, every registration"}, + "repository": map[string]any{"type": "string", "description": "only the registrations of this repository, owner/name"}, + "history": map[string]any{"type": "boolean", "description": "with every version of the one module"}, + "next": map[string]any{"type": "string", "description": "the cursor the previous page answered"}}}, + Run: read}, {Name: meshcatalog.Seat + "." + meshcatalog.VerbRegister, Run: act(verbRegister, true)}, {Name: meshcatalog.Seat + "." + meshcatalog.VerbPolicy, Run: act(verbPolicy, true)}, {Name: meshcatalog.Seat + "." + meshcatalog.VerbWatch, Run: act(verbWatch, true)}, diff --git a/modules/mesh-delivery-indexer/internal/index/registrations_tools_test.go b/modules/mesh-delivery-indexer/internal/index/registrations_tools_test.go index f9be83ff9..98b77ccbd 100644 --- a/modules/mesh-delivery-indexer/internal/index/registrations_tools_test.go +++ b/modules/mesh-delivery-indexer/internal/index/registrations_tools_test.go @@ -4,12 +4,28 @@ import ( "context" "encoding/json" "errors" + "fmt" + "slices" + "strings" "testing" "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + stdio "git.novox.be/novox/mesh-sdk/go" ) +// pagedStore is a store whose Page answers the pages given, by cursor, and records the cursors asked. +type pagedStore struct { + Store + pages map[string]stdio.StatePage + asked []string +} + +func (p *pagedStore) Page(cursor string, _ int, _ time.Duration) (stdio.StatePage, error) { + p.asked = append(p.asked, cursor) + return p.pages[cursor], nil +} + // Design 47 G A-370: the verb registrations answers from the state at once; it never asks the indexer (whose take-in may be // running), but for a module's history, which the indexer reads from its stream. func TestTheRegistrationsVerbIsAnsweredFromItsStateNotAskedOfTheIndexer(t *testing.T) { @@ -41,4 +57,72 @@ func TestTheRegistrationsVerbIsAnsweredFromItsStateNotAskedOfTheIndexer(t *testi if _, err := tool(map[string]any{"module": "a", "history": true}); err == nil || asked != 1 { t.Errorf("a history is asked of the indexer: err %v, asked %d", err, asked) } + + // Design 47 H2 §806: indexer_registrations answers as the verb, filtered by module and repository while it reads, at most + // 768 KB a page, capped with the cursor next; a state that pages is read from the bus's cursor. + t.Run("filtered and paged", func(t *testing.T) { + ctx := context.Background() + big := MemoryStore() + var want []string + for i := range 12 { + origin := "novox/mesh-catalog" + if i%3 == 0 { + origin = "novox/mesh-sdk" + } + module := fmt.Sprintf("m%02d", i) + data, _ := json.Marshal(meshcatalog.Registration{Module: module, Origin: origin, Path: strings.Repeat("p", 100<<10)}) + if err := big.Create(ctx, RegistrationKey(module), data); err != nil { + t.Fatal(err) + } + if origin == "novox/mesh-catalog" { + want = append(want, module) + } + } + named := func(tools []stdio.Tool) func(map[string]any) (any, error) { + for _, tl := range tools { + if tl.Name == RegistrationsTool { + return tl.Run + } + } + t.Fatalf("no tool %s", RegistrationsTool) + return nil + } + run := named(RegistrationTools(big, MemoryStore(), ask, time.Now, nil)) + var got []string + pages := 0 + for next := ""; ; pages++ { + raw, err := run(map[string]any{"repository": "novox/mesh-catalog", "next": next}) + a, _ := raw.(meshcatalog.RegistrationsAnswer) + if err != nil || a.Capped != (a.Next != "") { + t.Fatalf("page %d = %+v (%v), want capped exactly when a cursor follows", pages, a, err) + } + for _, r := range a.Registrations { + got = append(got, r.Module) + } + if next = a.Next; next == "" { + break + } + } + if !slices.Equal(got, want) || pages == 0 { + t.Errorf("the pages hold %v in %d pages after the first, want %v over more than one page", got, pages, want) + } + if a, _ := run(map[string]any{"module": "m00", "repository": "novox/mesh-catalog"}); len(a.(meshcatalog.RegistrationsAnswer).Registrations) != 0 { + t.Errorf("m00 of novox/mesh-sdk answered for the repository novox/mesh-catalog: %+v", a) + } + reg := func(module, origin string) stdio.StateEntry { + data, _ := json.Marshal(meshcatalog.Registration{Module: module, Origin: origin}) + return stdio.StateEntry{Key: RegistrationKey(module), Value: data} + } + pager := &pagedStore{Store: MemoryStore(), pages: map[string]stdio.StatePage{ + "": {Entries: []stdio.StateEntry{reg("a", "novox/x"), reg("b", "novox/y")}, Next: "c1"}, + "c1": {Entries: []stdio.StateEntry{reg("c", "novox/x")}}}} + run = named(RegistrationTools(pager, MemoryStore(), ask, time.Now, nil)) + first, _ := run(map[string]any{"repository": "novox/x"}) + second, _ := run(map[string]any{"repository": "novox/x", "next": "c1"}) + f, g := first.(meshcatalog.RegistrationsAnswer), second.(meshcatalog.RegistrationsAnswer) + if len(f.Registrations) != 1 || f.Registrations[0].Module != "a" || f.Next != "c1" || !f.Capped || + len(g.Registrations) != 1 || g.Registrations[0].Module != "c" || g.Next != "" || !slices.Equal(pager.asked, []string{"", "c1"}) { + t.Errorf("the paged read answered %+v then %+v, asking %v, want a then c of novox/x by the bus's cursor", f, g, pager.asked) + } + }) } diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 99f0974c0..ac6d421df 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -209,8 +209,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool, index.EntryIndexTool, index.EntryPinsTool, index.NamedDropTool, // design 47 G §675, H-A-204 index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.PinDropTool, index.PendingTool, index.TakeInPendingTool, index.TakeInAgainTool, index.TakeInSkipTool, index.RunsTool, index.RunDropTool, // design 47 G §667, §8, §579, §64 (A-194) - index.DryRunsTool) // design 47 H2 §804 - if !reflect.DeepEqual(m.Tools, want) || len(want) != 45 || !slices.Contains(m.Tools, "modules") { + index.DryRunsTool, index.RegistrationsTool) // design 47 H2 §804, §806 + if !reflect.DeepEqual(m.Tools, want) || len(want) != 46 || !slices.Contains(m.Tools, "modules") { t.Errorf("tools = %v, want %v", m.Tools, want) } // Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section. diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 8926fa796..b6a7943b6 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -71,7 +71,8 @@ "indexer_take_in_skip", "indexer_runs", "indexer_run_drop", - "indexer_dry_runs" + "indexer_dry_runs", + "indexer_registrations" ], "consumes": [ "go-package-registry.published",