A-479: the seat mesh-catalog's verbs answer from memory, one read of the trunk view per call
The tools bundle's verbs module, modules, provides and dependents read the modules and the module graph key by key from the newest trunk revision, and each key listed the whole catalog-revisions bucket again (6 s per call at 11 000 objects, 35 s with 12 callers); seat-definitions read its state once per seat. The view is read once per call (SourceValues), the seat definitions come from a watched copy, a manifest is read once by its digest. Builds: design 47 G (the seat mesh-catalog's verbs served by the indexer's tools bundle; ADR 0604 §2, ADR 0580 clause 1)
This commit is contained in:
@@ -15,24 +15,17 @@ import (
|
||||
|
||||
// catalogRead reads the indexer's view of the trunk (its modules and module graph, from the newest catalog graph
|
||||
// revision) and its manifests state for the seat's verbs.
|
||||
func catalogRead(modules views.Source, manifests stdio.KeptState, nodes views.Source, seatDefinitions, marks views.Source) index.CatalogRead {
|
||||
func catalogRead(modules views.Source, manifests *stdio.WatchedStates, nodes views.Source, seatDefinitions, marks views.Source) index.CatalogRead {
|
||||
return index.CatalogRead{
|
||||
SeatDefinitions: seatDefinitions, Marks: marks,
|
||||
Modules: func() (map[string]meshcatalog.ModulesByModuleValue, error) {
|
||||
keys, err := modules.Keys()
|
||||
values, err := index.SourceValues(modules)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]meshcatalog.ModulesByModuleValue{}
|
||||
for _, k := range keys {
|
||||
e, ok, err := modules.Get(k)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("modules.%s: %w", k, err)
|
||||
}
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if m, err := meshcatalogclient.DecodeModule(e.Value); err == nil {
|
||||
for k, v := range values {
|
||||
if m, err := meshcatalogclient.DecodeModule(v); err == nil {
|
||||
out[k] = m
|
||||
}
|
||||
}
|
||||
@@ -43,31 +36,25 @@ func catalogRead(modules views.Source, manifests stdio.KeptState, nodes views.So
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
e, err := manifests.Get(key)
|
||||
// A manifest never changes under its digest: read once from the state and kept.
|
||||
value, ok, err := manifests.Once("manifests", key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if e == nil {
|
||||
if !ok {
|
||||
return "", fmt.Errorf("the manifests state holds no %s", digest)
|
||||
}
|
||||
kept, err := meshcatalogclient.DecodeManifest(e.Value)
|
||||
kept, err := meshcatalogclient.DecodeManifest(value)
|
||||
return kept.Manifest, err
|
||||
},
|
||||
Graph: func() (graph.Graph, error) {
|
||||
keys, err := nodes.Keys()
|
||||
values, err := index.SourceValues(nodes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
g := graph.Graph{}
|
||||
for _, k := range keys {
|
||||
e, ok, err := nodes.Get(k)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("graph.%s: %w", k, err)
|
||||
}
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if n, err := graph.Decode(e.Value); err == nil {
|
||||
for k, v := range values {
|
||||
if n, err := graph.Decode(v); err == nil {
|
||||
g[k] = n
|
||||
}
|
||||
}
|
||||
@@ -77,7 +64,28 @@ func catalogRead(modules views.Source, manifests stdio.KeptState, nodes views.So
|
||||
}
|
||||
|
||||
// catalogTools are the seat's verbs over the indexer's own states, served beside the indexer's own tools, since the indexer's manifest claims the seat (ADR 0604 §2).
|
||||
func catalogTools(state func(string) views.Source) []stdio.Tool {
|
||||
return index.CatalogTools(catalogRead(state("modules"), stdio.State("manifests"), state(graph.State),
|
||||
keptSource(index.SeatDefinitionsState), keptSource(views.MarksState)))
|
||||
// Every state a verb reads whole is answered from memory (a watched copy, or the view of the trunk read once per call), so
|
||||
// a call costs no round trip per key: four node-engines ask every module at once (the verb module read the catalog-revisions
|
||||
// bucket's listing once per module, and seat-definitions the state once per seat).
|
||||
func catalogTools(state func(string) views.Source, watched *stdio.WatchedStates) []stdio.Tool {
|
||||
return index.CatalogTools(catalogRead(state("modules"), watched, state(graph.State),
|
||||
watchedSource{watched, index.SeatDefinitionsState}, keptSource(views.MarksState)))
|
||||
}
|
||||
|
||||
// watchedSource is a state as its watched copy holds it (stdio.WatchedStates), as a Source; it offers every value in one read.
|
||||
type watchedSource struct {
|
||||
w *stdio.WatchedStates
|
||||
state string
|
||||
}
|
||||
|
||||
func (s watchedSource) Keys() ([]string, error) { return s.w.Keys(s.state) }
|
||||
|
||||
func (s watchedSource) Get(key string) (*views.Entry, bool, error) {
|
||||
v, ok, err := s.w.Get(s.state, key)
|
||||
if err != nil || !ok {
|
||||
return nil, false, err
|
||||
}
|
||||
return &views.Entry{Value: v}, true, nil
|
||||
}
|
||||
|
||||
func (s watchedSource) Values() (map[string][]byte, error) { return s.w.Values(s.state) }
|
||||
|
||||
@@ -289,7 +289,8 @@ func main() {
|
||||
trunk := index.TrunkStates{Objects: revisions, Under: keptSource, Kept: &index.KeptTrunk{Say: func(format string, a ...any) {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] "+format+"\n", a...)
|
||||
}}}
|
||||
tools = append(tools, catalogTools(trunk.State)...)
|
||||
watched := stdio.NewWatchedStates()
|
||||
tools = append(tools, catalogTools(trunk.State, watched)...)
|
||||
// The seat mesh-catalog's verb revision, a first view of the catalog half answered in this runtime of its own to exactly
|
||||
// the parts whose entry declares it (novox/hq ADR 0663).
|
||||
tools = append(tools, index.RevisionTool(revisions, os.Getenv(stdio.ServedAs)), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)}, keptRead{stdio.State(index.NamedState)}), index.VersionsTool(keptRead{stdio.State(index.VersionsState)}),
|
||||
@@ -322,7 +323,7 @@ func main() {
|
||||
_, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.FingerprintsAgainTool, Verb: "fingerprints-again"})
|
||||
return err
|
||||
}, 10*time.Second)...)
|
||||
tools = append(tools, configReadersTool(stdio.NewWatchedStates(), trunk.State("modules")), busStateTool(trunk.State), requirementsTool(keptSource),
|
||||
tools = append(tools, configReadersTool(watched, trunk.State("modules")), busStateTool(trunk.State), requirementsTool(keptSource),
|
||||
packageUsersTool(trunk.State), seatDefinitionTool(keptSource), outOfStepTool(os.Getenv(stdio.ServedAs), keptSource))
|
||||
if err := stdio.Serve("", tools); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err)
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
|
||||
// listingObjects counts the listings of the bucket: each costs the whole bucket (56 ms at 11 000 objects, more through the runtime).
|
||||
type listingObjects struct {
|
||||
*memObjects
|
||||
listings atomic.Int64
|
||||
}
|
||||
|
||||
func (l *listingObjects) Names(ctx context.Context) ([]string, error) {
|
||||
l.listings.Add(1)
|
||||
return l.memObjects.Names(ctx)
|
||||
}
|
||||
|
||||
// A seat verb that answers from the whole view of the trunk (modules, provides, dependents, module) reads the catalog-revisions
|
||||
// bucket's listing once per call, never once per module: the node-engines ask `module` for every module at once, and 33 asks
|
||||
// in 5 minutes passed the tool runner's 30 s (2026-11-17).
|
||||
func TestTrunkViewIsReadOnceForAWholeAnswer(t *testing.T) {
|
||||
objects := &listingObjects{memObjects: newMemObjects(0)}
|
||||
const modules = 40
|
||||
var refs []meshcatalog.EntryRef
|
||||
for m := 0; m < modules; m++ {
|
||||
name := fmt.Sprintf("mod%02d", m)
|
||||
digest := fmt.Sprintf("%064x", m)
|
||||
e := meshcatalog.CatalogEntry{Module: name, Repository: "o/r", Path: "m/" + name, Commit: "c", Fp: "src2:" + strings.Repeat("a", 64),
|
||||
Manifest: map[string]any{"name": name}, ManifestDigest: "sha256:" + digest}
|
||||
b, _ := json.Marshal(e)
|
||||
if err := objects.Put(context.Background(), EntryObject(digest), b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
refs = append(refs, meshcatalog.EntryRef{Module: name, Digest: digest})
|
||||
}
|
||||
b, _ := json.Marshal(meshcatalog.CatalogRevision{Kind: "trunk", Revision: 1, Entries: refs})
|
||||
if err := objects.Put(context.Background(), "trunk.1", b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ts := TrunkStates{Objects: objects, Under: func(string) views.Source { return nil }, Kept: &KeptTrunk{}}
|
||||
for _, state := range []string{"modules", "graph"} {
|
||||
objects.listings.Store(0)
|
||||
values, err := SourceValues(ts.State(state))
|
||||
if err != nil || len(values) != modules {
|
||||
t.Fatalf("the %s of the view = %d values (%v), want %d", state, len(values), err, modules)
|
||||
}
|
||||
if n := objects.listings.Load(); n > 2 {
|
||||
t.Errorf("a whole read of the %s listed the bucket %d times, want at most 2 (once per call, not once per module)", state, n)
|
||||
}
|
||||
}
|
||||
if _, err := meshcatalogclient.DecodeModule(mustValue(t, ts)); err != nil {
|
||||
t.Errorf("a value of the view does not decode: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustValue(t *testing.T, ts TrunkStates) []byte {
|
||||
t.Helper()
|
||||
v, err := SourceValues(ts.State("modules"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return v["mod00"]
|
||||
}
|
||||
@@ -588,6 +588,57 @@ func (r revisionSource) Get(key string) (*views.Entry, bool, error) {
|
||||
return &views.Entry{Value: value, Revision: uint64(at)}, true, nil
|
||||
}
|
||||
|
||||
// Values is every value of the modules (or the module graph) of the newest trunk revision from one read of it: a seat verb
|
||||
// that answers from the whole view asks which revision is newest once, never once per key (each ask lists the whole
|
||||
// catalog-revisions bucket: 6 s for the 110 modules of a bucket of 11 000 objects, and 35 s with 12 callers at once).
|
||||
func (r revisionSource) Values() (map[string][]byte, error) {
|
||||
s, _, err := r.t.read(r.nodes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx := context.Background()
|
||||
keys, err := s.Keys(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make(map[string][]byte, len(keys))
|
||||
for _, k := range keys {
|
||||
value, _, found, err := s.Get(ctx, k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if found {
|
||||
out[k] = value
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// SourceValues is every key's value of a source: in one read where the source offers it (the view of the trunk, a
|
||||
// watched copy of a state), else a get per key.
|
||||
func SourceValues(src views.Source) (map[string][]byte, error) {
|
||||
if all, ok := src.(interface {
|
||||
Values() (map[string][]byte, error)
|
||||
}); ok {
|
||||
return all.Values()
|
||||
}
|
||||
keys, err := src.Keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make(map[string][]byte, len(keys))
|
||||
for _, k := range keys {
|
||||
e, ok, err := src.Get(k)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", k, err)
|
||||
}
|
||||
if ok && e != nil {
|
||||
out[k] = e.Value
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Page is the values in key order after a key from one read of the revision, as many as fit maxBytes and at least one
|
||||
// (views.Pager): a first view's page reads the revision once, never once per key.
|
||||
func (r revisionSource) Page(after string, maxBytes int, _ time.Duration) ([]views.KeyedEntry, string, error) {
|
||||
|
||||
Reference in New Issue
Block a user