diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go index 52b5aea2..535c8689 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/catalog.go @@ -15,24 +15,17 @@ import ( // catalogRead reads the indexer's view of the trunk (its modules and module graph, from the newest catalog graph // revision) and its manifests state for the seat's verbs. -func catalogRead(modules views.Source, manifests stdio.KeptState, nodes views.Source, seatDefinitions, marks views.Source) index.CatalogRead { +func catalogRead(modules views.Source, manifests *stdio.WatchedStates, nodes views.Source, seatDefinitions, marks views.Source) index.CatalogRead { return index.CatalogRead{ SeatDefinitions: seatDefinitions, Marks: marks, Modules: func() (map[string]meshcatalog.ModulesByModuleValue, error) { - keys, err := modules.Keys() + values, err := index.SourceValues(modules) if err != nil { return nil, err } out := map[string]meshcatalog.ModulesByModuleValue{} - for _, k := range keys { - e, ok, err := modules.Get(k) - if err != nil { - return nil, fmt.Errorf("modules.%s: %w", k, err) - } - if !ok { - continue - } - if m, err := meshcatalogclient.DecodeModule(e.Value); err == nil { + for k, v := range values { + if m, err := meshcatalogclient.DecodeModule(v); err == nil { out[k] = m } } @@ -43,31 +36,25 @@ func catalogRead(modules views.Source, manifests stdio.KeptState, nodes views.So if err != nil { return "", err } - e, err := manifests.Get(key) + // A manifest never changes under its digest: read once from the state and kept. + value, ok, err := manifests.Once("manifests", key) if err != nil { return "", err } - if e == nil { + if !ok { return "", fmt.Errorf("the manifests state holds no %s", digest) } - kept, err := meshcatalogclient.DecodeManifest(e.Value) + kept, err := meshcatalogclient.DecodeManifest(value) return kept.Manifest, err }, Graph: func() (graph.Graph, error) { - keys, err := nodes.Keys() + values, err := index.SourceValues(nodes) if err != nil { return nil, err } g := graph.Graph{} - for _, k := range keys { - e, ok, err := nodes.Get(k) - if err != nil { - return nil, fmt.Errorf("graph.%s: %w", k, err) - } - if !ok { - continue - } - if n, err := graph.Decode(e.Value); err == nil { + for k, v := range values { + if n, err := graph.Decode(v); err == nil { g[k] = n } } @@ -77,7 +64,28 @@ func catalogRead(modules views.Source, manifests stdio.KeptState, nodes views.So } // catalogTools are the seat's verbs over the indexer's own states, served beside the indexer's own tools, since the indexer's manifest claims the seat (ADR 0604 ยง2). -func catalogTools(state func(string) views.Source) []stdio.Tool { - return index.CatalogTools(catalogRead(state("modules"), stdio.State("manifests"), state(graph.State), - keptSource(index.SeatDefinitionsState), keptSource(views.MarksState))) +// Every state a verb reads whole is answered from memory (a watched copy, or the view of the trunk read once per call), so +// a call costs no round trip per key: four node-engines ask every module at once (the verb module read the catalog-revisions +// bucket's listing once per module, and seat-definitions the state once per seat). +func catalogTools(state func(string) views.Source, watched *stdio.WatchedStates) []stdio.Tool { + return index.CatalogTools(catalogRead(state("modules"), watched, state(graph.State), + watchedSource{watched, index.SeatDefinitionsState}, keptSource(views.MarksState))) } + +// watchedSource is a state as its watched copy holds it (stdio.WatchedStates), as a Source; it offers every value in one read. +type watchedSource struct { + w *stdio.WatchedStates + state string +} + +func (s watchedSource) Keys() ([]string, error) { return s.w.Keys(s.state) } + +func (s watchedSource) Get(key string) (*views.Entry, bool, error) { + v, ok, err := s.w.Get(s.state, key) + if err != nil || !ok { + return nil, false, err + } + return &views.Entry{Value: v}, true, nil +} + +func (s watchedSource) Values() (map[string][]byte, error) { return s.w.Values(s.state) } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index 12599d11..33f9b653 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -289,7 +289,8 @@ func main() { trunk := index.TrunkStates{Objects: revisions, Under: keptSource, Kept: &index.KeptTrunk{Say: func(format string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] "+format+"\n", a...) }}} - tools = append(tools, catalogTools(trunk.State)...) + watched := stdio.NewWatchedStates() + tools = append(tools, catalogTools(trunk.State, watched)...) // The seat mesh-catalog's verb revision, a first view of the catalog half answered in this runtime of its own to exactly // the parts whose entry declares it (novox/hq ADR 0663). tools = append(tools, index.RevisionTool(revisions, os.Getenv(stdio.ServedAs)), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)}, keptRead{stdio.State(index.NamedState)}), index.VersionsTool(keptRead{stdio.State(index.VersionsState)}), @@ -322,7 +323,7 @@ func main() { _, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.FingerprintsAgainTool, Verb: "fingerprints-again"}) return err }, 10*time.Second)...) - tools = append(tools, configReadersTool(stdio.NewWatchedStates(), trunk.State("modules")), busStateTool(trunk.State), requirementsTool(keptSource), + tools = append(tools, configReadersTool(watched, trunk.State("modules")), busStateTool(trunk.State), requirementsTool(keptSource), packageUsersTool(trunk.State), seatDefinitionTool(keptSource), outOfStepTool(os.Getenv(stdio.ServedAs), keptSource)) if err := stdio.Serve("", tools); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err) diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_calls_test.go b/modules/mesh-delivery-indexer/internal/index/catalog_calls_test.go new file mode 100644 index 00000000..445ebad6 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/catalog_calls_test.go @@ -0,0 +1,72 @@ +package index + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "sync/atomic" + "testing" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/views" +) + +// listingObjects counts the listings of the bucket: each costs the whole bucket (56 ms at 11 000 objects, more through the runtime). +type listingObjects struct { + *memObjects + listings atomic.Int64 +} + +func (l *listingObjects) Names(ctx context.Context) ([]string, error) { + l.listings.Add(1) + return l.memObjects.Names(ctx) +} + +// A seat verb that answers from the whole view of the trunk (modules, provides, dependents, module) reads the catalog-revisions +// bucket's listing once per call, never once per module: the node-engines ask `module` for every module at once, and 33 asks +// in 5 minutes passed the tool runner's 30 s (2026-11-17). +func TestTrunkViewIsReadOnceForAWholeAnswer(t *testing.T) { + objects := &listingObjects{memObjects: newMemObjects(0)} + const modules = 40 + var refs []meshcatalog.EntryRef + for m := 0; m < modules; m++ { + name := fmt.Sprintf("mod%02d", m) + digest := fmt.Sprintf("%064x", m) + e := meshcatalog.CatalogEntry{Module: name, Repository: "o/r", Path: "m/" + name, Commit: "c", Fp: "src2:" + strings.Repeat("a", 64), + Manifest: map[string]any{"name": name}, ManifestDigest: "sha256:" + digest} + b, _ := json.Marshal(e) + if err := objects.Put(context.Background(), EntryObject(digest), b); err != nil { + t.Fatal(err) + } + refs = append(refs, meshcatalog.EntryRef{Module: name, Digest: digest}) + } + b, _ := json.Marshal(meshcatalog.CatalogRevision{Kind: "trunk", Revision: 1, Entries: refs}) + if err := objects.Put(context.Background(), "trunk.1", b); err != nil { + t.Fatal(err) + } + ts := TrunkStates{Objects: objects, Under: func(string) views.Source { return nil }, Kept: &KeptTrunk{}} + for _, state := range []string{"modules", "graph"} { + objects.listings.Store(0) + values, err := SourceValues(ts.State(state)) + if err != nil || len(values) != modules { + t.Fatalf("the %s of the view = %d values (%v), want %d", state, len(values), err, modules) + } + if n := objects.listings.Load(); n > 2 { + t.Errorf("a whole read of the %s listed the bucket %d times, want at most 2 (once per call, not once per module)", state, n) + } + } + if _, err := meshcatalogclient.DecodeModule(mustValue(t, ts)); err != nil { + t.Errorf("a value of the view does not decode: %v", err) + } +} + +func mustValue(t *testing.T, ts TrunkStates) []byte { + t.Helper() + v, err := SourceValues(ts.State("modules")) + if err != nil { + t.Fatal(err) + } + return v["mod00"] +} diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index ca51d308..73dfd69b 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -588,6 +588,57 @@ func (r revisionSource) Get(key string) (*views.Entry, bool, error) { return &views.Entry{Value: value, Revision: uint64(at)}, true, nil } +// Values is every value of the modules (or the module graph) of the newest trunk revision from one read of it: a seat verb +// that answers from the whole view asks which revision is newest once, never once per key (each ask lists the whole +// catalog-revisions bucket: 6 s for the 110 modules of a bucket of 11 000 objects, and 35 s with 12 callers at once). +func (r revisionSource) Values() (map[string][]byte, error) { + s, _, err := r.t.read(r.nodes) + if err != nil { + return nil, err + } + ctx := context.Background() + keys, err := s.Keys(ctx) + if err != nil { + return nil, err + } + out := make(map[string][]byte, len(keys)) + for _, k := range keys { + value, _, found, err := s.Get(ctx, k) + if err != nil { + return nil, err + } + if found { + out[k] = value + } + } + return out, nil +} + +// SourceValues is every key's value of a source: in one read where the source offers it (the view of the trunk, a +// watched copy of a state), else a get per key. +func SourceValues(src views.Source) (map[string][]byte, error) { + if all, ok := src.(interface { + Values() (map[string][]byte, error) + }); ok { + return all.Values() + } + keys, err := src.Keys() + if err != nil { + return nil, err + } + out := make(map[string][]byte, len(keys)) + for _, k := range keys { + e, ok, err := src.Get(k) + if err != nil { + return nil, fmt.Errorf("%s: %w", k, err) + } + if ok && e != nil { + out[k] = e.Value + } + } + return out, nil +} + // Page is the values in key order after a key from one read of the revision, as many as fit maxBytes and at least one // (views.Pager): a first view's page reads the revision once, never once per key. func (r revisionSource) Page(after string, maxBytes int, _ time.Duration) ([]views.KeyedEntry, string, error) {