I5: the registrations are read in one call, filtered and paged

The verb registrations and the new tool indexer_registrations filter by
module and repository while they read, at most 768 KB a page, capped with
the cursor next: the tools bundle reads the state from the bus's cursor
(mesh/state.page), a store that pages none by its keys in order; total is
said when one page holds the whole read.

Builds: design 47 H2 §806, §811, §779
This commit is contained in:
2026-11-22 01:12:04 +01:00
parent d719a69b7d
commit cc5f803f87
6 changed files with 187 additions and 34 deletions
@@ -156,6 +156,11 @@ func (k keptRead) Get(_ context.Context, key string) ([]byte, uint64, bool, erro
func (k keptRead) Keys(context.Context) ([]string, error) { return k.s.Keys() }
// Page reads the state a page at a time from the bus's cursor (design 47 H2 §779): the registrations are read so (§806).
func (k keptRead) Page(cursor string, maxBytes int, budget time.Duration) (stdio.StatePage, error) {
return k.s.Page(cursor, maxBytes, budget)
}
func (keptRead) Create(context.Context, string, []byte) error { return errReadOnly }
func (keptRead) Update(context.Context, string, []byte, uint64) error { return errReadOnly }
@@ -16,7 +16,10 @@ import (
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
gitseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/views"
)
// A module's registration, and the trusted change of one (design 47 G §2 to §4, §10, §342, §343, §667): the indexer keeps each
@@ -169,15 +172,34 @@ func (r *Registry) Registration(ctx context.Context, module string) (meshcatalog
return reg, true, nil
}
// Registrations is the verb registrations (§4): every registration, or one module's, with its history.
// RegistrationsTool is the indexer's tool of the registrations (design 47 H2 §806): the verb registrations' answer, by hand.
const RegistrationsTool = "indexer_registrations"
// StatePager reads a state a page at a time, in the order the bus wrote it, after an opaque cursor (mesh-sdk
// KeptState.Page, design 47 H2 §779).
type StatePager interface {
Page(cursor string, maxBytes int, budget time.Duration) (stdio.StatePage, error)
}
// keyCursor is the cursor of a store read by its keys (a store that pages none): after this key, in key order.
const keyCursor = "key:"
// RegistrationsAnswer is the verb registrations and the tool indexer_registrations (§4; design 47 H2 §806, §811): every
// registration, or one module's with its history, filtered by module and repository while it reads, at most views.PageBytes
// a page, capped with the cursor next when more is held; a store that pages (StatePager) is read from the bus's cursor,
// any other by its keys in order. total is how many the filters hold, said when one page holds the whole read.
func (r *Registry) RegistrationsAnswer(ctx context.Context, in meshcatalog.RegistrationsInput) (meshcatalog.RegistrationsAnswer, error) {
out := meshcatalog.RegistrationsAnswer{Registrations: []meshcatalog.Registration{}}
repository := trunk.RepositoryName(strings.TrimSpace(in.Repository))
keep := func(reg meshcatalog.Registration) bool {
return (in.Module == "" || reg.Module == in.Module) && (repository == "" || trunk.SameRepository(reg.Origin, repository))
}
if in.Module != "" {
reg, found, err := r.Registration(ctx, in.Module)
if err != nil {
return out, err
}
if found {
if found && keep(reg) {
out.Registrations = append(out.Registrations, reg)
}
if in.History && r.HistoryOf != nil {
@@ -185,22 +207,53 @@ func (r *Registry) RegistrationsAnswer(ctx context.Context, in meshcatalog.Regis
return out, fmt.Errorf("the history of %s cannot be read: %w", in.Module, err)
}
}
out.Total = int64(len(out.Registrations))
return out, nil
}
all, err := r.Registrations.Keys(ctx)
if err != nil {
return out, err
match := func(value []byte) (meshcatalog.Registration, bool) {
var reg meshcatalog.Registration
return reg, json.Unmarshal(value, &reg) == nil && keep(reg)
}
slices.Sort(all)
for _, k := range all {
v, _, found, err := r.Registrations.Get(ctx, k)
if p, ok := r.Registrations.(StatePager); ok {
page, err := p.Page(in.Next, views.PageBytes, views.PageBudget)
if err != nil {
return out, err
}
var reg meshcatalog.Registration
if found && json.Unmarshal(v, &reg) == nil {
out.Registrations = append(out.Registrations, reg)
// A page holds at most views.PageBytes, so what the filters keep of it always fits one answer.
for _, e := range page.Entries {
if reg, ok := match(e.Value); ok {
out.Registrations = append(out.Registrations, reg)
}
}
out.Next, out.Capped = page.Next, page.Next != ""
} else {
all, err := r.Registrations.Keys(ctx)
if err != nil {
return out, err
}
slices.Sort(all)
after, size := strings.TrimPrefix(in.Next, keyCursor), 0
for _, k := range all {
if in.Next != "" && k <= after {
continue
}
v, _, found, err := r.Registrations.Get(ctx, k)
if err != nil {
return out, err
}
reg, ok := match(v)
if !found || !ok {
continue
}
if size += len(k) + len(v); size > views.PageBytes && len(out.Registrations) > 0 {
out.Capped, out.Next = true, keyCursor+after
return out, nil
}
out.Registrations, after = append(out.Registrations, reg), k
}
}
if in.Next == "" && !out.Capped {
out.Total = int64(len(out.Registrations))
}
return out, nil
}
@@ -43,27 +43,37 @@ func RegistrationTools(registrations, proposals Store, ask func(key string, valu
// in, never queued behind a running take-in (design 47 G A-370). Only a module's history is read from the stream the
// indexer keeps, so that is still asked of the indexer.
viaIndexer := act(ActRegistrations, true)
read := func(args map[string]any) (any, error) {
var in meshcatalog.RegistrationsInput
clean := map[string]any{}
for k, v := range args {
if k != "caller" && !strings.HasPrefix(k, "_") {
clean[k] = v
}
}
raw, err := json.Marshal(clean)
if err == nil {
err = json.Unmarshal(raw, &in)
}
if err != nil {
return nil, err
}
if in.History {
return viaIndexer(args)
}
return (&Registry{Registrations: registrations}).RegistrationsAnswer(context.Background(), in)
}
return []stdio.Tool{
{Name: meshcatalog.Seat + "." + meshcatalog.VerbRegistrations, Run: func(args map[string]any) (any, error) {
var in meshcatalog.RegistrationsInput
clean := map[string]any{}
for k, v := range args {
if k != "caller" && !strings.HasPrefix(k, "_") {
clean[k] = v
}
}
raw, err := json.Marshal(clean)
if err == nil {
err = json.Unmarshal(raw, &in)
}
if err != nil {
return nil, err
}
if in.History {
return viaIndexer(args)
}
return (&Registry{Registrations: registrations}).RegistrationsAnswer(context.Background(), in)
}},
{Name: meshcatalog.Seat + "." + meshcatalog.VerbRegistrations, Run: read},
{Name: RegistrationsTool, Description: "The registrations (design 47 H2 §806), as the seat mesh-catalog's verb registrations answers them: " +
"every module's and every source-only one, filtered by module or repository (owner/name) while it reads, at most 768 KB a page " +
"(capped), the rest read from the cursor next; one module's with every version (history).",
Input: map[string]any{"type": "object", "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "one module; absent, every registration"},
"repository": map[string]any{"type": "string", "description": "only the registrations of this repository, owner/name"},
"history": map[string]any{"type": "boolean", "description": "with every version of the one module"},
"next": map[string]any{"type": "string", "description": "the cursor the previous page answered"}}},
Run: read},
{Name: meshcatalog.Seat + "." + meshcatalog.VerbRegister, Run: act(verbRegister, true)},
{Name: meshcatalog.Seat + "." + meshcatalog.VerbPolicy, Run: act(verbPolicy, true)},
{Name: meshcatalog.Seat + "." + meshcatalog.VerbWatch, Run: act(verbWatch, true)},
@@ -4,12 +4,28 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// pagedStore is a store whose Page answers the pages given, by cursor, and records the cursors asked.
type pagedStore struct {
Store
pages map[string]stdio.StatePage
asked []string
}
func (p *pagedStore) Page(cursor string, _ int, _ time.Duration) (stdio.StatePage, error) {
p.asked = append(p.asked, cursor)
return p.pages[cursor], nil
}
// Design 47 G A-370: the verb registrations answers from the state at once; it never asks the indexer (whose take-in may be
// running), but for a module's history, which the indexer reads from its stream.
func TestTheRegistrationsVerbIsAnsweredFromItsStateNotAskedOfTheIndexer(t *testing.T) {
@@ -41,4 +57,72 @@ func TestTheRegistrationsVerbIsAnsweredFromItsStateNotAskedOfTheIndexer(t *testi
if _, err := tool(map[string]any{"module": "a", "history": true}); err == nil || asked != 1 {
t.Errorf("a history is asked of the indexer: err %v, asked %d", err, asked)
}
// Design 47 H2 §806: indexer_registrations answers as the verb, filtered by module and repository while it reads, at most
// 768 KB a page, capped with the cursor next; a state that pages is read from the bus's cursor.
t.Run("filtered and paged", func(t *testing.T) {
ctx := context.Background()
big := MemoryStore()
var want []string
for i := range 12 {
origin := "novox/mesh-catalog"
if i%3 == 0 {
origin = "novox/mesh-sdk"
}
module := fmt.Sprintf("m%02d", i)
data, _ := json.Marshal(meshcatalog.Registration{Module: module, Origin: origin, Path: strings.Repeat("p", 100<<10)})
if err := big.Create(ctx, RegistrationKey(module), data); err != nil {
t.Fatal(err)
}
if origin == "novox/mesh-catalog" {
want = append(want, module)
}
}
named := func(tools []stdio.Tool) func(map[string]any) (any, error) {
for _, tl := range tools {
if tl.Name == RegistrationsTool {
return tl.Run
}
}
t.Fatalf("no tool %s", RegistrationsTool)
return nil
}
run := named(RegistrationTools(big, MemoryStore(), ask, time.Now, nil))
var got []string
pages := 0
for next := ""; ; pages++ {
raw, err := run(map[string]any{"repository": "novox/mesh-catalog", "next": next})
a, _ := raw.(meshcatalog.RegistrationsAnswer)
if err != nil || a.Capped != (a.Next != "") {
t.Fatalf("page %d = %+v (%v), want capped exactly when a cursor follows", pages, a, err)
}
for _, r := range a.Registrations {
got = append(got, r.Module)
}
if next = a.Next; next == "" {
break
}
}
if !slices.Equal(got, want) || pages == 0 {
t.Errorf("the pages hold %v in %d pages after the first, want %v over more than one page", got, pages, want)
}
if a, _ := run(map[string]any{"module": "m00", "repository": "novox/mesh-catalog"}); len(a.(meshcatalog.RegistrationsAnswer).Registrations) != 0 {
t.Errorf("m00 of novox/mesh-sdk answered for the repository novox/mesh-catalog: %+v", a)
}
reg := func(module, origin string) stdio.StateEntry {
data, _ := json.Marshal(meshcatalog.Registration{Module: module, Origin: origin})
return stdio.StateEntry{Key: RegistrationKey(module), Value: data}
}
pager := &pagedStore{Store: MemoryStore(), pages: map[string]stdio.StatePage{
"": {Entries: []stdio.StateEntry{reg("a", "novox/x"), reg("b", "novox/y")}, Next: "c1"},
"c1": {Entries: []stdio.StateEntry{reg("c", "novox/x")}}}}
run = named(RegistrationTools(pager, MemoryStore(), ask, time.Now, nil))
first, _ := run(map[string]any{"repository": "novox/x"})
second, _ := run(map[string]any{"repository": "novox/x", "next": "c1"})
f, g := first.(meshcatalog.RegistrationsAnswer), second.(meshcatalog.RegistrationsAnswer)
if len(f.Registrations) != 1 || f.Registrations[0].Module != "a" || f.Next != "c1" || !f.Capped ||
len(g.Registrations) != 1 || g.Registrations[0].Module != "c" || g.Next != "" || !slices.Equal(pager.asked, []string{"", "c1"}) {
t.Errorf("the paged read answered %+v then %+v, asking %v, want a then c of novox/x by the bus's cursor", f, g, pager.asked)
}
})
}
@@ -209,8 +209,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool,
index.EntryIndexTool, index.EntryPinsTool, index.NamedDropTool, // design 47 G §675, H-A-204
index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.PinDropTool, index.PendingTool, index.TakeInPendingTool, index.TakeInAgainTool, index.TakeInSkipTool, index.RunsTool, index.RunDropTool, // design 47 G §667, §8, §579, §64 (A-194)
index.DryRunsTool) // design 47 H2 §804
if !reflect.DeepEqual(m.Tools, want) || len(want) != 45 || !slices.Contains(m.Tools, "modules") {
index.DryRunsTool, index.RegistrationsTool) // design 47 H2 §804, §806
if !reflect.DeepEqual(m.Tools, want) || len(want) != 46 || !slices.Contains(m.Tools, "modules") {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
// Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section.
+2 -1
View File
@@ -71,7 +71,8 @@
"indexer_take_in_skip",
"indexer_runs",
"indexer_run_drop",
"indexer_dry_runs"
"indexer_dry_runs",
"indexer_registrations"
],
"consumes": [
"go-package-registry.published",