Indexer tools: keep the view of the trunk while its revision stays the newest, and page it: every key re-read the whole revision, so propose timed out on a first view of about 11 s a page

The assigner says each indexer view it reads, how many pages and how long, and a page that does not answer.
This commit is contained in:
2026-11-14 20:59:31 +01:00
parent 4ee5b45cb8
commit 3cdd4943f0
4 changed files with 102 additions and 3 deletions
@@ -7,6 +7,7 @@ import (
"fmt"
"maps"
"mesh-delivery-assigner/internal/buckets"
"os"
"slices"
"sort"
"strings"
@@ -67,13 +68,17 @@ func indexerView(ask func(string, any) (json.RawMessage, error), state string) (
}
page := views.AskVia(ask, tool)
out := map[string]json.RawMessage{}
started, pages := time.Now(), 0
for after := ""; ; {
a, err := page(context.Background(), after)
if err != nil {
fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: %s does not answer page %d after %s: %v\n", tool, pages+1, time.Since(started).Round(time.Millisecond), err)
return nil, fmt.Errorf("%s does not answer: %w", tool, err)
}
pages++
maps.Copy(out, a.Values)
if a.Next == "" || a.Next == after {
fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: %s read, %d values in %d pages, %s\n", tool, len(out), pages, time.Since(started).Round(time.Millisecond))
return out, nil
}
after = a.Next
@@ -286,7 +286,9 @@ func main() {
tools := Tools(asks, time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState)), acts)
// The modules and the module graph are the newest catalog graph revision's (milestone 47/G): revision.go.
revisions := bundleObjects{busstate.ObjectsOf(index.CatalogRevisionsObjects)}
trunk := index.TrunkStates{Objects: revisions, Under: keptSource}
trunk := index.TrunkStates{Objects: revisions, Under: keptSource, Kept: &index.KeptTrunk{Say: func(format string, a ...any) {
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] "+format+"\n", a...)
}}}
tools = append(tools, catalogTools(trunk.State)...)
// The seat mesh-catalog's verb revision, a first view of the catalog half answered in this runtime of its own to exactly
// the parts whose entry declares it (novox/hq ADR 0663).
@@ -8,6 +8,7 @@ import (
"slices"
"strconv"
"strings"
"sync"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
@@ -489,11 +490,25 @@ var ErrNoRevision = errors.New("view-waits: the indexer keeps no catalog graph r
type TrunkStates struct {
Objects Objects
Under func(string) views.Source
// Kept, when set, holds the view of the newest revision read, so a read asks only which revision is newest and reads
// the view again only when a newer one is kept: without it every key read the whole revision again, one object per
// module, and a first view of the modules took about 11 s a page. Nil reads the view at every read.
Kept *KeptTrunk
}
// KeptTrunk is the view of the trunk as last read, at its revision, shared by every read of a TrunkStates; Say, when set,
// says each read of a revision and how long it took.
type KeptTrunk struct {
Say func(format string, a ...any)
mu sync.Mutex
at int64
mods Store
held Store
}
// read is the view's modules and nodes, and the revision they were read at; none kept yet is ErrNoRevision.
func (t TrunkStates) read(nodes bool) (Store, int64, error) {
mods, held, at, err := TrunkView(context.Background(), t.Objects)
mods, held, at, err := t.view()
if err == nil && at == 0 {
err = ErrNoRevision
}
@@ -503,6 +518,38 @@ func (t TrunkStates) read(nodes bool) (Store, int64, error) {
return mods, at, err
}
// view is the view of the newest trunk revision: the one kept when it is still the newest, else read whole and kept.
func (t TrunkStates) view() (modules, nodes Store, at int64, err error) {
ctx := context.Background()
if t.Kept == nil {
return TrunkView(ctx, t.Objects)
}
k := t.Kept
newest, any, err := newestOf(ctx, t.Objects, TrunkBranch)
if err != nil || !any {
return newOverlay(nil), newOverlay(nil), 0, err
}
k.mu.Lock()
defer k.mu.Unlock()
if k.mods != nil && k.at == newest.Revision {
return k.mods, k.held, k.at, nil
}
started := time.Now()
mods, held, at, err := TrunkView(ctx, t.Objects)
if err != nil {
if k.Say != nil {
k.Say("the view of the trunk: revision %d cannot be read after %s: %v", newest.Revision, time.Since(started).Round(time.Millisecond), err)
}
return nil, nil, 0, err
}
if k.Say != nil {
n, _ := mods.Keys(ctx)
k.Say("the view of the trunk: read revision %d (was %d), %d modules, in %s", at, k.at, len(n), time.Since(started).Round(time.Millisecond))
}
k.mods, k.held, k.at = mods, held, at
return mods, held, at, nil
}
// state is the source of a state by its name: modules and graph from the revision, their marks at its number, every
// other the indexer's own state.
func (t TrunkStates) State(name string) views.Source {
@@ -541,6 +588,40 @@ func (r revisionSource) Get(key string) (*views.Entry, bool, error) {
return &views.Entry{Value: value, Revision: uint64(at)}, true, nil
}
// Page is the values in key order after a key from one read of the revision, as many as fit maxBytes and at least one
// (views.Pager): a first view's page reads the revision once, never once per key.
func (r revisionSource) Page(after string, maxBytes int, _ time.Duration) ([]views.KeyedEntry, string, error) {
s, at, err := r.t.read(r.nodes)
if err != nil {
return nil, "", err
}
ctx := context.Background()
keys, err := s.Keys(ctx)
if err != nil {
return nil, "", err
}
slices.Sort(keys)
out, size := []views.KeyedEntry{}, 0
for _, k := range keys {
if after != "" && k <= after {
continue
}
value, _, found, err := s.Get(ctx, k)
if err != nil {
return nil, "", err
}
if !found {
continue
}
if len(out) > 0 && size+len(k)+len(value) > maxBytes {
return out, out[len(out)-1].Key, nil
}
out = append(out, views.KeyedEntry{Key: k, Entry: views.Entry{Value: value, Revision: uint64(at)}})
size += len(k) + len(value)
}
return out, "", nil
}
// markSource answers the mark of modules and graph as the newest trunk revision's number, which catalog-revision says
// the changes after; every other mark is the views family's.
type markSource struct {
@@ -608,8 +608,19 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { //
if e, ok, err := trunkStates.State(views.MarksState).Get("graph"); err != nil || !ok || !strings.Contains(string(e.Value), `"revision":2`) {
t.Errorf("the bundle's mark of graph = %+v %v (%v), want revision 2", e, ok, err)
}
// A kept view reads the revision once while it stays the newest, and its first view answers in pages of values, never
// one revision read per key (about 11 s a page live, 2026-11-14).
reads := 0
keptView := TrunkStates{Objects: first.Revisions, Under: func(string) views.Source { return nil }, Kept: &KeptTrunk{Say: func(string, ...any) { reads++ }}}
if a, err := views.FirstView(testSelf, ModulesState, keptView.State(ModulesState), keptView.State(views.MarksState), "", time.Now()); err != nil ||
a.Revision != 2 || len(a.Values) != len(builderModules(t)) || a.Next != "" {
t.Errorf("the kept first view modules answered %d values at %d, next %q (%v), want every module at revision 2 in one page", len(a.Values), a.Revision, a.Next, err)
}
if _, _, err := keptView.State("graph").Get("avahi"); err != nil || reads != 1 {
t.Errorf("the kept view read the revision %d times (%v), want once", reads, err)
}
if _, err := (TrunkStates{Objects: newMemObjects(0)}).State("graph").Keys(); !errors.Is(err, ErrNoRevision) {
t.Errorf("the bundle's graph with no revision kept answered %v, want it to wait", err)
t.Errorf("the bundle's graph with no revision keptView answered %v, want it to wait", err)
}
// W4.4, ADR 0663: the seat mesh-catalog's verb revision answers, from the indexer's own objects, exactly the parts whose
// entry in the newest trunk revision declares it, the holder's own tools too (A-401); a candidate's by its group and heads; none