A-479: the seat mesh-catalog's verbs answer from memory, one read of the trunk view per call

The tools bundle's verbs module, modules, provides and dependents read the modules and the module graph key by key from
the newest trunk revision, and each key listed the whole catalog-revisions bucket again (6 s per call at 11 000 objects,
35 s with 12 callers); seat-definitions read its state once per seat. The view is read once per call (SourceValues), the
seat definitions come from a watched copy, a manifest is read once by its digest.

Builds: design 47 G (the seat mesh-catalog's verbs served by the indexer's tools bundle; ADR 0604 §2, ADR 0580 clause 1)
This commit is contained in:
2026-11-17 22:43:08 +01:00
parent d006132c2a
commit ea106ea8b9
4 changed files with 161 additions and 29 deletions
@@ -15,24 +15,17 @@ import (
// catalogRead reads the indexer's view of the trunk (its modules and module graph, from the newest catalog graph // catalogRead reads the indexer's view of the trunk (its modules and module graph, from the newest catalog graph
// revision) and its manifests state for the seat's verbs. // revision) and its manifests state for the seat's verbs.
func catalogRead(modules views.Source, manifests stdio.KeptState, nodes views.Source, seatDefinitions, marks views.Source) index.CatalogRead { func catalogRead(modules views.Source, manifests *stdio.WatchedStates, nodes views.Source, seatDefinitions, marks views.Source) index.CatalogRead {
return index.CatalogRead{ return index.CatalogRead{
SeatDefinitions: seatDefinitions, Marks: marks, SeatDefinitions: seatDefinitions, Marks: marks,
Modules: func() (map[string]meshcatalog.ModulesByModuleValue, error) { Modules: func() (map[string]meshcatalog.ModulesByModuleValue, error) {
keys, err := modules.Keys() values, err := index.SourceValues(modules)
if err != nil { if err != nil {
return nil, err return nil, err
} }
out := map[string]meshcatalog.ModulesByModuleValue{} out := map[string]meshcatalog.ModulesByModuleValue{}
for _, k := range keys { for k, v := range values {
e, ok, err := modules.Get(k) if m, err := meshcatalogclient.DecodeModule(v); err == nil {
if err != nil {
return nil, fmt.Errorf("modules.%s: %w", k, err)
}
if !ok {
continue
}
if m, err := meshcatalogclient.DecodeModule(e.Value); err == nil {
out[k] = m out[k] = m
} }
} }
@@ -43,31 +36,25 @@ func catalogRead(modules views.Source, manifests stdio.KeptState, nodes views.So
if err != nil { if err != nil {
return "", err return "", err
} }
e, err := manifests.Get(key) // A manifest never changes under its digest: read once from the state and kept.
value, ok, err := manifests.Once("manifests", key)
if err != nil { if err != nil {
return "", err return "", err
} }
if e == nil { if !ok {
return "", fmt.Errorf("the manifests state holds no %s", digest) return "", fmt.Errorf("the manifests state holds no %s", digest)
} }
kept, err := meshcatalogclient.DecodeManifest(e.Value) kept, err := meshcatalogclient.DecodeManifest(value)
return kept.Manifest, err return kept.Manifest, err
}, },
Graph: func() (graph.Graph, error) { Graph: func() (graph.Graph, error) {
keys, err := nodes.Keys() values, err := index.SourceValues(nodes)
if err != nil { if err != nil {
return nil, err return nil, err
} }
g := graph.Graph{} g := graph.Graph{}
for _, k := range keys { for k, v := range values {
e, ok, err := nodes.Get(k) if n, err := graph.Decode(v); err == nil {
if err != nil {
return nil, fmt.Errorf("graph.%s: %w", k, err)
}
if !ok {
continue
}
if n, err := graph.Decode(e.Value); err == nil {
g[k] = n g[k] = n
} }
} }
@@ -77,7 +64,28 @@ func catalogRead(modules views.Source, manifests stdio.KeptState, nodes views.So
} }
// catalogTools are the seat's verbs over the indexer's own states, served beside the indexer's own tools, since the indexer's manifest claims the seat (ADR 0604 §2). // catalogTools are the seat's verbs over the indexer's own states, served beside the indexer's own tools, since the indexer's manifest claims the seat (ADR 0604 §2).
func catalogTools(state func(string) views.Source) []stdio.Tool { // Every state a verb reads whole is answered from memory (a watched copy, or the view of the trunk read once per call), so
return index.CatalogTools(catalogRead(state("modules"), stdio.State("manifests"), state(graph.State), // a call costs no round trip per key: four node-engines ask every module at once (the verb module read the catalog-revisions
keptSource(index.SeatDefinitionsState), keptSource(views.MarksState))) // bucket's listing once per module, and seat-definitions the state once per seat).
func catalogTools(state func(string) views.Source, watched *stdio.WatchedStates) []stdio.Tool {
return index.CatalogTools(catalogRead(state("modules"), watched, state(graph.State),
watchedSource{watched, index.SeatDefinitionsState}, keptSource(views.MarksState)))
} }
// watchedSource is a state as its watched copy holds it (stdio.WatchedStates), as a Source; it offers every value in one read.
type watchedSource struct {
w *stdio.WatchedStates
state string
}
func (s watchedSource) Keys() ([]string, error) { return s.w.Keys(s.state) }
func (s watchedSource) Get(key string) (*views.Entry, bool, error) {
v, ok, err := s.w.Get(s.state, key)
if err != nil || !ok {
return nil, false, err
}
return &views.Entry{Value: v}, true, nil
}
func (s watchedSource) Values() (map[string][]byte, error) { return s.w.Values(s.state) }
@@ -289,7 +289,8 @@ func main() {
trunk := index.TrunkStates{Objects: revisions, Under: keptSource, Kept: &index.KeptTrunk{Say: func(format string, a ...any) { trunk := index.TrunkStates{Objects: revisions, Under: keptSource, Kept: &index.KeptTrunk{Say: func(format string, a ...any) {
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] "+format+"\n", a...) fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] "+format+"\n", a...)
}}} }}}
tools = append(tools, catalogTools(trunk.State)...) watched := stdio.NewWatchedStates()
tools = append(tools, catalogTools(trunk.State, watched)...)
// The seat mesh-catalog's verb revision, a first view of the catalog half answered in this runtime of its own to exactly // The seat mesh-catalog's verb revision, a first view of the catalog half answered in this runtime of its own to exactly
// the parts whose entry declares it (novox/hq ADR 0663). // the parts whose entry declares it (novox/hq ADR 0663).
tools = append(tools, index.RevisionTool(revisions, os.Getenv(stdio.ServedAs)), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)}, keptRead{stdio.State(index.NamedState)}), index.VersionsTool(keptRead{stdio.State(index.VersionsState)}), tools = append(tools, index.RevisionTool(revisions, os.Getenv(stdio.ServedAs)), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)}, keptRead{stdio.State(index.NamedState)}), index.VersionsTool(keptRead{stdio.State(index.VersionsState)}),
@@ -322,7 +323,7 @@ func main() {
_, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.FingerprintsAgainTool, Verb: "fingerprints-again"}) _, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.FingerprintsAgainTool, Verb: "fingerprints-again"})
return err return err
}, 10*time.Second)...) }, 10*time.Second)...)
tools = append(tools, configReadersTool(stdio.NewWatchedStates(), trunk.State("modules")), busStateTool(trunk.State), requirementsTool(keptSource), tools = append(tools, configReadersTool(watched, trunk.State("modules")), busStateTool(trunk.State), requirementsTool(keptSource),
packageUsersTool(trunk.State), seatDefinitionTool(keptSource), outOfStepTool(os.Getenv(stdio.ServedAs), keptSource)) packageUsersTool(trunk.State), seatDefinitionTool(keptSource), outOfStepTool(os.Getenv(stdio.ServedAs), keptSource))
if err := stdio.Serve("", tools); err != nil { if err := stdio.Serve("", tools); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err) fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err)
@@ -0,0 +1,72 @@
package index
import (
"context"
"encoding/json"
"fmt"
"strings"
"sync/atomic"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/views"
)
// listingObjects counts the listings of the bucket: each costs the whole bucket (56 ms at 11 000 objects, more through the runtime).
type listingObjects struct {
*memObjects
listings atomic.Int64
}
func (l *listingObjects) Names(ctx context.Context) ([]string, error) {
l.listings.Add(1)
return l.memObjects.Names(ctx)
}
// A seat verb that answers from the whole view of the trunk (modules, provides, dependents, module) reads the catalog-revisions
// bucket's listing once per call, never once per module: the node-engines ask `module` for every module at once, and 33 asks
// in 5 minutes passed the tool runner's 30 s (2026-11-17).
func TestTrunkViewIsReadOnceForAWholeAnswer(t *testing.T) {
objects := &listingObjects{memObjects: newMemObjects(0)}
const modules = 40
var refs []meshcatalog.EntryRef
for m := 0; m < modules; m++ {
name := fmt.Sprintf("mod%02d", m)
digest := fmt.Sprintf("%064x", m)
e := meshcatalog.CatalogEntry{Module: name, Repository: "o/r", Path: "m/" + name, Commit: "c", Fp: "src2:" + strings.Repeat("a", 64),
Manifest: map[string]any{"name": name}, ManifestDigest: "sha256:" + digest}
b, _ := json.Marshal(e)
if err := objects.Put(context.Background(), EntryObject(digest), b); err != nil {
t.Fatal(err)
}
refs = append(refs, meshcatalog.EntryRef{Module: name, Digest: digest})
}
b, _ := json.Marshal(meshcatalog.CatalogRevision{Kind: "trunk", Revision: 1, Entries: refs})
if err := objects.Put(context.Background(), "trunk.1", b); err != nil {
t.Fatal(err)
}
ts := TrunkStates{Objects: objects, Under: func(string) views.Source { return nil }, Kept: &KeptTrunk{}}
for _, state := range []string{"modules", "graph"} {
objects.listings.Store(0)
values, err := SourceValues(ts.State(state))
if err != nil || len(values) != modules {
t.Fatalf("the %s of the view = %d values (%v), want %d", state, len(values), err, modules)
}
if n := objects.listings.Load(); n > 2 {
t.Errorf("a whole read of the %s listed the bucket %d times, want at most 2 (once per call, not once per module)", state, n)
}
}
if _, err := meshcatalogclient.DecodeModule(mustValue(t, ts)); err != nil {
t.Errorf("a value of the view does not decode: %v", err)
}
}
func mustValue(t *testing.T, ts TrunkStates) []byte {
t.Helper()
v, err := SourceValues(ts.State("modules"))
if err != nil {
t.Fatal(err)
}
return v["mod00"]
}
@@ -588,6 +588,57 @@ func (r revisionSource) Get(key string) (*views.Entry, bool, error) {
return &views.Entry{Value: value, Revision: uint64(at)}, true, nil return &views.Entry{Value: value, Revision: uint64(at)}, true, nil
} }
// Values is every value of the modules (or the module graph) of the newest trunk revision from one read of it: a seat verb
// that answers from the whole view asks which revision is newest once, never once per key (each ask lists the whole
// catalog-revisions bucket: 6 s for the 110 modules of a bucket of 11 000 objects, and 35 s with 12 callers at once).
func (r revisionSource) Values() (map[string][]byte, error) {
s, _, err := r.t.read(r.nodes)
if err != nil {
return nil, err
}
ctx := context.Background()
keys, err := s.Keys(ctx)
if err != nil {
return nil, err
}
out := make(map[string][]byte, len(keys))
for _, k := range keys {
value, _, found, err := s.Get(ctx, k)
if err != nil {
return nil, err
}
if found {
out[k] = value
}
}
return out, nil
}
// SourceValues is every key's value of a source: in one read where the source offers it (the view of the trunk, a
// watched copy of a state), else a get per key.
func SourceValues(src views.Source) (map[string][]byte, error) {
if all, ok := src.(interface {
Values() (map[string][]byte, error)
}); ok {
return all.Values()
}
keys, err := src.Keys()
if err != nil {
return nil, err
}
out := make(map[string][]byte, len(keys))
for _, k := range keys {
e, ok, err := src.Get(k)
if err != nil {
return nil, fmt.Errorf("%s: %w", k, err)
}
if ok && e != nil {
out[k] = e.Value
}
}
return out, nil
}
// Page is the values in key order after a key from one read of the revision, as many as fit maxBytes and at least one // Page is the values in key order after a key from one read of the revision, as many as fit maxBytes and at least one
// (views.Pager): a first view's page reads the revision once, never once per key. // (views.Pager): a first view's page reads the revision once, never once per key.
func (r revisionSource) Page(after string, maxBytes int, _ time.Duration) ([]views.KeyedEntry, string, error) { func (r revisionSource) Page(after string, maxBytes int, _ time.Duration) ([]views.KeyedEntry, string, error) {