From 3cdd4943f0d37c81eeb6a6209c54b385199b3252 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 14 Nov 2026 20:59:31 +0100 Subject: [PATCH] Indexer tools: keep the view of the trunk while its revision stays the newest, and page it: every key re-read the whole revision, so propose timed out on a first view of about 11 s a page The assigner says each indexer view it reads, how many pages and how long, and a page that does not answer. --- .../cmd/mesh-delivery-assigner/seat_tools.go | 5 ++ .../cmd/mesh-delivery-indexer-tools/main.go | 4 +- .../internal/index/catalog_revisions.go | 83 ++++++++++++++++++- .../internal/index/toolchains_test.go | 13 ++- 4 files changed, 102 insertions(+), 3 deletions(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go index 0673524f..abee2787 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go @@ -7,6 +7,7 @@ import ( "fmt" "maps" "mesh-delivery-assigner/internal/buckets" + "os" "slices" "sort" "strings" @@ -67,13 +68,17 @@ func indexerView(ask func(string, any) (json.RawMessage, error), state string) ( } page := views.AskVia(ask, tool) out := map[string]json.RawMessage{} + started, pages := time.Now(), 0 for after := ""; ; { a, err := page(context.Background(), after) if err != nil { + fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: %s does not answer page %d after %s: %v\n", tool, pages+1, time.Since(started).Round(time.Millisecond), err) return nil, fmt.Errorf("%s does not answer: %w", tool, err) } + pages++ maps.Copy(out, a.Values) if a.Next == "" || a.Next == after { + fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: %s read, %d values in %d pages, %s\n", tool, len(out), pages, time.Since(started).Round(time.Millisecond)) return out, nil } after = a.Next diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index d08d624c..664447b7 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -286,7 +286,9 @@ func main() { tools := Tools(asks, time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState)), acts) // The modules and the module graph are the newest catalog graph revision's (milestone 47/G): revision.go. revisions := bundleObjects{busstate.ObjectsOf(index.CatalogRevisionsObjects)} - trunk := index.TrunkStates{Objects: revisions, Under: keptSource} + trunk := index.TrunkStates{Objects: revisions, Under: keptSource, Kept: &index.KeptTrunk{Say: func(format string, a ...any) { + fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] "+format+"\n", a...) + }}} tools = append(tools, catalogTools(trunk.State)...) // The seat mesh-catalog's verb revision, a first view of the catalog half answered in this runtime of its own to exactly // the parts whose entry declares it (novox/hq ADR 0663). diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index c65774f4..d78477b7 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -8,6 +8,7 @@ import ( "slices" "strconv" "strings" + "sync" "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" @@ -489,11 +490,25 @@ var ErrNoRevision = errors.New("view-waits: the indexer keeps no catalog graph r type TrunkStates struct { Objects Objects Under func(string) views.Source + // Kept, when set, holds the view of the newest revision read, so a read asks only which revision is newest and reads + // the view again only when a newer one is kept: without it every key read the whole revision again, one object per + // module, and a first view of the modules took about 11 s a page. Nil reads the view at every read. + Kept *KeptTrunk +} + +// KeptTrunk is the view of the trunk as last read, at its revision, shared by every read of a TrunkStates; Say, when set, +// says each read of a revision and how long it took. +type KeptTrunk struct { + Say func(format string, a ...any) + mu sync.Mutex + at int64 + mods Store + held Store } // read is the view's modules and nodes, and the revision they were read at; none kept yet is ErrNoRevision. func (t TrunkStates) read(nodes bool) (Store, int64, error) { - mods, held, at, err := TrunkView(context.Background(), t.Objects) + mods, held, at, err := t.view() if err == nil && at == 0 { err = ErrNoRevision } @@ -503,6 +518,38 @@ func (t TrunkStates) read(nodes bool) (Store, int64, error) { return mods, at, err } +// view is the view of the newest trunk revision: the one kept when it is still the newest, else read whole and kept. +func (t TrunkStates) view() (modules, nodes Store, at int64, err error) { + ctx := context.Background() + if t.Kept == nil { + return TrunkView(ctx, t.Objects) + } + k := t.Kept + newest, any, err := newestOf(ctx, t.Objects, TrunkBranch) + if err != nil || !any { + return newOverlay(nil), newOverlay(nil), 0, err + } + k.mu.Lock() + defer k.mu.Unlock() + if k.mods != nil && k.at == newest.Revision { + return k.mods, k.held, k.at, nil + } + started := time.Now() + mods, held, at, err := TrunkView(ctx, t.Objects) + if err != nil { + if k.Say != nil { + k.Say("the view of the trunk: revision %d cannot be read after %s: %v", newest.Revision, time.Since(started).Round(time.Millisecond), err) + } + return nil, nil, 0, err + } + if k.Say != nil { + n, _ := mods.Keys(ctx) + k.Say("the view of the trunk: read revision %d (was %d), %d modules, in %s", at, k.at, len(n), time.Since(started).Round(time.Millisecond)) + } + k.mods, k.held, k.at = mods, held, at + return mods, held, at, nil +} + // state is the source of a state by its name: modules and graph from the revision, their marks at its number, every // other the indexer's own state. func (t TrunkStates) State(name string) views.Source { @@ -541,6 +588,40 @@ func (r revisionSource) Get(key string) (*views.Entry, bool, error) { return &views.Entry{Value: value, Revision: uint64(at)}, true, nil } +// Page is the values in key order after a key from one read of the revision, as many as fit maxBytes and at least one +// (views.Pager): a first view's page reads the revision once, never once per key. +func (r revisionSource) Page(after string, maxBytes int, _ time.Duration) ([]views.KeyedEntry, string, error) { + s, at, err := r.t.read(r.nodes) + if err != nil { + return nil, "", err + } + ctx := context.Background() + keys, err := s.Keys(ctx) + if err != nil { + return nil, "", err + } + slices.Sort(keys) + out, size := []views.KeyedEntry{}, 0 + for _, k := range keys { + if after != "" && k <= after { + continue + } + value, _, found, err := s.Get(ctx, k) + if err != nil { + return nil, "", err + } + if !found { + continue + } + if len(out) > 0 && size+len(k)+len(value) > maxBytes { + return out, out[len(out)-1].Key, nil + } + out = append(out, views.KeyedEntry{Key: k, Entry: views.Entry{Value: value, Revision: uint64(at)}}) + size += len(k) + len(value) + } + return out, "", nil +} + // markSource answers the mark of modules and graph as the newest trunk revision's number, which catalog-revision says // the changes after; every other mark is the views family's. type markSource struct { diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index a46ee5f0..2b1ab74e 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -608,8 +608,19 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // if e, ok, err := trunkStates.State(views.MarksState).Get("graph"); err != nil || !ok || !strings.Contains(string(e.Value), `"revision":2`) { t.Errorf("the bundle's mark of graph = %+v %v (%v), want revision 2", e, ok, err) } + // A kept view reads the revision once while it stays the newest, and its first view answers in pages of values, never + // one revision read per key (about 11 s a page live, 2026-11-14). + reads := 0 + keptView := TrunkStates{Objects: first.Revisions, Under: func(string) views.Source { return nil }, Kept: &KeptTrunk{Say: func(string, ...any) { reads++ }}} + if a, err := views.FirstView(testSelf, ModulesState, keptView.State(ModulesState), keptView.State(views.MarksState), "", time.Now()); err != nil || + a.Revision != 2 || len(a.Values) != len(builderModules(t)) || a.Next != "" { + t.Errorf("the kept first view modules answered %d values at %d, next %q (%v), want every module at revision 2 in one page", len(a.Values), a.Revision, a.Next, err) + } + if _, _, err := keptView.State("graph").Get("avahi"); err != nil || reads != 1 { + t.Errorf("the kept view read the revision %d times (%v), want once", reads, err) + } if _, err := (TrunkStates{Objects: newMemObjects(0)}).State("graph").Keys(); !errors.Is(err, ErrNoRevision) { - t.Errorf("the bundle's graph with no revision kept answered %v, want it to wait", err) + t.Errorf("the bundle's graph with no revision keptView answered %v, want it to wait", err) } // W4.4, ADR 0663: the seat mesh-catalog's verb revision answers, from the indexer's own objects, exactly the parts whose // entry in the newest trunk revision declares it, the holder's own tools too (A-401); a candidate's by its group and heads; none