I5: the registrations are read in one call, filtered and paged
The verb registrations and the new tool indexer_registrations filter by module and repository while they read, at most 768 KB a page, capped with the cursor next: the tools bundle reads the state from the bus's cursor (mesh/state.page), a store that pages none by its keys in order; total is said when one page holds the whole read. Builds: design 47 H2 §806, §811, §779
This commit is contained in:
@@ -156,6 +156,11 @@ func (k keptRead) Get(_ context.Context, key string) ([]byte, uint64, bool, erro
|
||||
|
||||
func (k keptRead) Keys(context.Context) ([]string, error) { return k.s.Keys() }
|
||||
|
||||
// Page reads the state a page at a time from the bus's cursor (design 47 H2 §779): the registrations are read so (§806).
|
||||
func (k keptRead) Page(cursor string, maxBytes int, budget time.Duration) (stdio.StatePage, error) {
|
||||
return k.s.Page(cursor, maxBytes, budget)
|
||||
}
|
||||
|
||||
func (keptRead) Create(context.Context, string, []byte) error { return errReadOnly }
|
||||
|
||||
func (keptRead) Update(context.Context, string, []byte, uint64) error { return errReadOnly }
|
||||
|
||||
@@ -16,7 +16,10 @@ import (
|
||||
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
||||
gitseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
|
||||
// A module's registration, and the trusted change of one (design 47 G §2 to §4, §10, §342, §343, §667): the indexer keeps each
|
||||
@@ -169,15 +172,34 @@ func (r *Registry) Registration(ctx context.Context, module string) (meshcatalog
|
||||
return reg, true, nil
|
||||
}
|
||||
|
||||
// Registrations is the verb registrations (§4): every registration, or one module's, with its history.
|
||||
// RegistrationsTool is the indexer's tool of the registrations (design 47 H2 §806): the verb registrations' answer, by hand.
|
||||
const RegistrationsTool = "indexer_registrations"
|
||||
|
||||
// StatePager reads a state a page at a time, in the order the bus wrote it, after an opaque cursor (mesh-sdk
|
||||
// KeptState.Page, design 47 H2 §779).
|
||||
type StatePager interface {
|
||||
Page(cursor string, maxBytes int, budget time.Duration) (stdio.StatePage, error)
|
||||
}
|
||||
|
||||
// keyCursor is the cursor of a store read by its keys (a store that pages none): after this key, in key order.
|
||||
const keyCursor = "key:"
|
||||
|
||||
// RegistrationsAnswer is the verb registrations and the tool indexer_registrations (§4; design 47 H2 §806, §811): every
|
||||
// registration, or one module's with its history, filtered by module and repository while it reads, at most views.PageBytes
|
||||
// a page, capped with the cursor next when more is held; a store that pages (StatePager) is read from the bus's cursor,
|
||||
// any other by its keys in order. total is how many the filters hold, said when one page holds the whole read.
|
||||
func (r *Registry) RegistrationsAnswer(ctx context.Context, in meshcatalog.RegistrationsInput) (meshcatalog.RegistrationsAnswer, error) {
|
||||
out := meshcatalog.RegistrationsAnswer{Registrations: []meshcatalog.Registration{}}
|
||||
repository := trunk.RepositoryName(strings.TrimSpace(in.Repository))
|
||||
keep := func(reg meshcatalog.Registration) bool {
|
||||
return (in.Module == "" || reg.Module == in.Module) && (repository == "" || trunk.SameRepository(reg.Origin, repository))
|
||||
}
|
||||
if in.Module != "" {
|
||||
reg, found, err := r.Registration(ctx, in.Module)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
if found {
|
||||
if found && keep(reg) {
|
||||
out.Registrations = append(out.Registrations, reg)
|
||||
}
|
||||
if in.History && r.HistoryOf != nil {
|
||||
@@ -185,22 +207,53 @@ func (r *Registry) RegistrationsAnswer(ctx context.Context, in meshcatalog.Regis
|
||||
return out, fmt.Errorf("the history of %s cannot be read: %w", in.Module, err)
|
||||
}
|
||||
}
|
||||
out.Total = int64(len(out.Registrations))
|
||||
return out, nil
|
||||
}
|
||||
all, err := r.Registrations.Keys(ctx)
|
||||
if err != nil {
|
||||
return out, err
|
||||
match := func(value []byte) (meshcatalog.Registration, bool) {
|
||||
var reg meshcatalog.Registration
|
||||
return reg, json.Unmarshal(value, ®) == nil && keep(reg)
|
||||
}
|
||||
slices.Sort(all)
|
||||
for _, k := range all {
|
||||
v, _, found, err := r.Registrations.Get(ctx, k)
|
||||
if p, ok := r.Registrations.(StatePager); ok {
|
||||
page, err := p.Page(in.Next, views.PageBytes, views.PageBudget)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
var reg meshcatalog.Registration
|
||||
if found && json.Unmarshal(v, ®) == nil {
|
||||
out.Registrations = append(out.Registrations, reg)
|
||||
// A page holds at most views.PageBytes, so what the filters keep of it always fits one answer.
|
||||
for _, e := range page.Entries {
|
||||
if reg, ok := match(e.Value); ok {
|
||||
out.Registrations = append(out.Registrations, reg)
|
||||
}
|
||||
}
|
||||
out.Next, out.Capped = page.Next, page.Next != ""
|
||||
} else {
|
||||
all, err := r.Registrations.Keys(ctx)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
slices.Sort(all)
|
||||
after, size := strings.TrimPrefix(in.Next, keyCursor), 0
|
||||
for _, k := range all {
|
||||
if in.Next != "" && k <= after {
|
||||
continue
|
||||
}
|
||||
v, _, found, err := r.Registrations.Get(ctx, k)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
reg, ok := match(v)
|
||||
if !found || !ok {
|
||||
continue
|
||||
}
|
||||
if size += len(k) + len(v); size > views.PageBytes && len(out.Registrations) > 0 {
|
||||
out.Capped, out.Next = true, keyCursor+after
|
||||
return out, nil
|
||||
}
|
||||
out.Registrations, after = append(out.Registrations, reg), k
|
||||
}
|
||||
}
|
||||
if in.Next == "" && !out.Capped {
|
||||
out.Total = int64(len(out.Registrations))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -43,27 +43,37 @@ func RegistrationTools(registrations, proposals Store, ask func(key string, valu
|
||||
// in, never queued behind a running take-in (design 47 G A-370). Only a module's history is read from the stream the
|
||||
// indexer keeps, so that is still asked of the indexer.
|
||||
viaIndexer := act(ActRegistrations, true)
|
||||
read := func(args map[string]any) (any, error) {
|
||||
var in meshcatalog.RegistrationsInput
|
||||
clean := map[string]any{}
|
||||
for k, v := range args {
|
||||
if k != "caller" && !strings.HasPrefix(k, "_") {
|
||||
clean[k] = v
|
||||
}
|
||||
}
|
||||
raw, err := json.Marshal(clean)
|
||||
if err == nil {
|
||||
err = json.Unmarshal(raw, &in)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if in.History {
|
||||
return viaIndexer(args)
|
||||
}
|
||||
return (&Registry{Registrations: registrations}).RegistrationsAnswer(context.Background(), in)
|
||||
}
|
||||
return []stdio.Tool{
|
||||
{Name: meshcatalog.Seat + "." + meshcatalog.VerbRegistrations, Run: func(args map[string]any) (any, error) {
|
||||
var in meshcatalog.RegistrationsInput
|
||||
clean := map[string]any{}
|
||||
for k, v := range args {
|
||||
if k != "caller" && !strings.HasPrefix(k, "_") {
|
||||
clean[k] = v
|
||||
}
|
||||
}
|
||||
raw, err := json.Marshal(clean)
|
||||
if err == nil {
|
||||
err = json.Unmarshal(raw, &in)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if in.History {
|
||||
return viaIndexer(args)
|
||||
}
|
||||
return (&Registry{Registrations: registrations}).RegistrationsAnswer(context.Background(), in)
|
||||
}},
|
||||
{Name: meshcatalog.Seat + "." + meshcatalog.VerbRegistrations, Run: read},
|
||||
{Name: RegistrationsTool, Description: "The registrations (design 47 H2 §806), as the seat mesh-catalog's verb registrations answers them: " +
|
||||
"every module's and every source-only one, filtered by module or repository (owner/name) while it reads, at most 768 KB a page " +
|
||||
"(capped), the rest read from the cursor next; one module's with every version (history).",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "one module; absent, every registration"},
|
||||
"repository": map[string]any{"type": "string", "description": "only the registrations of this repository, owner/name"},
|
||||
"history": map[string]any{"type": "boolean", "description": "with every version of the one module"},
|
||||
"next": map[string]any{"type": "string", "description": "the cursor the previous page answered"}}},
|
||||
Run: read},
|
||||
{Name: meshcatalog.Seat + "." + meshcatalog.VerbRegister, Run: act(verbRegister, true)},
|
||||
{Name: meshcatalog.Seat + "." + meshcatalog.VerbPolicy, Run: act(verbPolicy, true)},
|
||||
{Name: meshcatalog.Seat + "." + meshcatalog.VerbWatch, Run: act(verbWatch, true)},
|
||||
|
||||
@@ -4,12 +4,28 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// pagedStore is a store whose Page answers the pages given, by cursor, and records the cursors asked.
|
||||
type pagedStore struct {
|
||||
Store
|
||||
pages map[string]stdio.StatePage
|
||||
asked []string
|
||||
}
|
||||
|
||||
func (p *pagedStore) Page(cursor string, _ int, _ time.Duration) (stdio.StatePage, error) {
|
||||
p.asked = append(p.asked, cursor)
|
||||
return p.pages[cursor], nil
|
||||
}
|
||||
|
||||
// Design 47 G A-370: the verb registrations answers from the state at once; it never asks the indexer (whose take-in may be
|
||||
// running), but for a module's history, which the indexer reads from its stream.
|
||||
func TestTheRegistrationsVerbIsAnsweredFromItsStateNotAskedOfTheIndexer(t *testing.T) {
|
||||
@@ -41,4 +57,72 @@ func TestTheRegistrationsVerbIsAnsweredFromItsStateNotAskedOfTheIndexer(t *testi
|
||||
if _, err := tool(map[string]any{"module": "a", "history": true}); err == nil || asked != 1 {
|
||||
t.Errorf("a history is asked of the indexer: err %v, asked %d", err, asked)
|
||||
}
|
||||
|
||||
// Design 47 H2 §806: indexer_registrations answers as the verb, filtered by module and repository while it reads, at most
|
||||
// 768 KB a page, capped with the cursor next; a state that pages is read from the bus's cursor.
|
||||
t.Run("filtered and paged", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
big := MemoryStore()
|
||||
var want []string
|
||||
for i := range 12 {
|
||||
origin := "novox/mesh-catalog"
|
||||
if i%3 == 0 {
|
||||
origin = "novox/mesh-sdk"
|
||||
}
|
||||
module := fmt.Sprintf("m%02d", i)
|
||||
data, _ := json.Marshal(meshcatalog.Registration{Module: module, Origin: origin, Path: strings.Repeat("p", 100<<10)})
|
||||
if err := big.Create(ctx, RegistrationKey(module), data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if origin == "novox/mesh-catalog" {
|
||||
want = append(want, module)
|
||||
}
|
||||
}
|
||||
named := func(tools []stdio.Tool) func(map[string]any) (any, error) {
|
||||
for _, tl := range tools {
|
||||
if tl.Name == RegistrationsTool {
|
||||
return tl.Run
|
||||
}
|
||||
}
|
||||
t.Fatalf("no tool %s", RegistrationsTool)
|
||||
return nil
|
||||
}
|
||||
run := named(RegistrationTools(big, MemoryStore(), ask, time.Now, nil))
|
||||
var got []string
|
||||
pages := 0
|
||||
for next := ""; ; pages++ {
|
||||
raw, err := run(map[string]any{"repository": "novox/mesh-catalog", "next": next})
|
||||
a, _ := raw.(meshcatalog.RegistrationsAnswer)
|
||||
if err != nil || a.Capped != (a.Next != "") {
|
||||
t.Fatalf("page %d = %+v (%v), want capped exactly when a cursor follows", pages, a, err)
|
||||
}
|
||||
for _, r := range a.Registrations {
|
||||
got = append(got, r.Module)
|
||||
}
|
||||
if next = a.Next; next == "" {
|
||||
break
|
||||
}
|
||||
}
|
||||
if !slices.Equal(got, want) || pages == 0 {
|
||||
t.Errorf("the pages hold %v in %d pages after the first, want %v over more than one page", got, pages, want)
|
||||
}
|
||||
if a, _ := run(map[string]any{"module": "m00", "repository": "novox/mesh-catalog"}); len(a.(meshcatalog.RegistrationsAnswer).Registrations) != 0 {
|
||||
t.Errorf("m00 of novox/mesh-sdk answered for the repository novox/mesh-catalog: %+v", a)
|
||||
}
|
||||
reg := func(module, origin string) stdio.StateEntry {
|
||||
data, _ := json.Marshal(meshcatalog.Registration{Module: module, Origin: origin})
|
||||
return stdio.StateEntry{Key: RegistrationKey(module), Value: data}
|
||||
}
|
||||
pager := &pagedStore{Store: MemoryStore(), pages: map[string]stdio.StatePage{
|
||||
"": {Entries: []stdio.StateEntry{reg("a", "novox/x"), reg("b", "novox/y")}, Next: "c1"},
|
||||
"c1": {Entries: []stdio.StateEntry{reg("c", "novox/x")}}}}
|
||||
run = named(RegistrationTools(pager, MemoryStore(), ask, time.Now, nil))
|
||||
first, _ := run(map[string]any{"repository": "novox/x"})
|
||||
second, _ := run(map[string]any{"repository": "novox/x", "next": "c1"})
|
||||
f, g := first.(meshcatalog.RegistrationsAnswer), second.(meshcatalog.RegistrationsAnswer)
|
||||
if len(f.Registrations) != 1 || f.Registrations[0].Module != "a" || f.Next != "c1" || !f.Capped ||
|
||||
len(g.Registrations) != 1 || g.Registrations[0].Module != "c" || g.Next != "" || !slices.Equal(pager.asked, []string{"", "c1"}) {
|
||||
t.Errorf("the paged read answered %+v then %+v, asking %v, want a then c of novox/x by the bus's cursor", f, g, pager.asked)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -209,8 +209,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool,
|
||||
index.EntryIndexTool, index.EntryPinsTool, index.NamedDropTool, // design 47 G §675, H-A-204
|
||||
index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.PinDropTool, index.PendingTool, index.TakeInPendingTool, index.TakeInAgainTool, index.TakeInSkipTool, index.RunsTool, index.RunDropTool, // design 47 G §667, §8, §579, §64 (A-194)
|
||||
index.DryRunsTool) // design 47 H2 §804
|
||||
if !reflect.DeepEqual(m.Tools, want) || len(want) != 45 || !slices.Contains(m.Tools, "modules") {
|
||||
index.DryRunsTool, index.RegistrationsTool) // design 47 H2 §804, §806
|
||||
if !reflect.DeepEqual(m.Tools, want) || len(want) != 46 || !slices.Contains(m.Tools, "modules") {
|
||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||
}
|
||||
// Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section.
|
||||
|
||||
@@ -71,7 +71,8 @@
|
||||
"indexer_take_in_skip",
|
||||
"indexer_runs",
|
||||
"indexer_run_drop",
|
||||
"indexer_dry_runs"
|
||||
"indexer_dry_runs",
|
||||
"indexer_registrations"
|
||||
],
|
||||
"consumes": [
|
||||
"go-package-registry.published",
|
||||
|
||||
Reference in New Issue
Block a user