diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go index 320d552a..6be29e84 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go @@ -108,25 +108,18 @@ type keyed struct { // assignments reads every node's assignments and the revision of its node key (its legacy key's before it is converted), // assembled from its keys in either form (mmac.Keys; research 072 row 37). func (s *seatTools) assignments() (map[string]mma.Assignments, map[string]uint64, error) { - keys, err := s.st.keys(mma.StateAssignments) + all, err := readAll(s.st, mma.StateAssignments) if err != nil { - return nil, nil, fmt.Errorf("assignments cannot be listed: %w", err) + return nil, nil, fmt.Errorf("assignments cannot be read: %w", err) } - sort.Strings(keys) + keys := slices.Sorted(maps.Keys(all)) held := mmac.NewKeys() var bad []string for _, key := range keys { if _, _, _, err := mmac.ParseKey(key); err != nil { continue } - raw, rev, found, err := s.st.get(mma.StateAssignments, key) - if err != nil { - return nil, nil, fmt.Errorf("assignments key %s cannot be read: %w", key, err) - } - if !found { - continue - } - if _, err := held.Put(key, raw, rev, false); err != nil { + if _, err := held.Put(key, all[key].raw, all[key].rev, false); err != nil { bad = append(bad, err.Error()) } } @@ -150,25 +143,18 @@ func (s *seatTools) lays() (func(string) bool, func(string) bool, error) { held[name] = true } } - haves, err := s.st.keys(haveState) + haves, err := readAll(s.st, haveState) if err != nil { - return nil, nil, fmt.Errorf("the nodes' records cannot be listed: %w", err) + return nil, nil, fmt.Errorf("the nodes' records cannot be read: %w", err) } recorded := map[string]bool{} - for _, k := range haves { + for k, v := range haves { node, _, isNode, err := nodeengineclient.ParseHaveKey(k) if err != nil || !isNode { continue } - asked := time.Now() - raw, _, found, err := s.st.get(haveState, k) - if err != nil || time.Since(asked) > 2*time.Second { - fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: the record %s.%s took %s to read: %v\n", haveState, k, time.Since(asked).Round(time.Millisecond), err) - } - if err == nil && found { - if h, err := nodeengineclient.DecodeHaveNode(raw); err == nil && h.Record != nil { - recorded[node] = true - } + if h, err := nodeengineclient.DecodeHaveNode(v.raw); err == nil && h.Record != nil { + recorded[node] = true } } return func(m string) bool { return held[m] }, func(n string) bool { return recorded[n] }, nil @@ -501,21 +487,17 @@ func (s *seatTools) doProposals(args map[string]any) (any, error) { if err != nil { return nil, fmt.Errorf("the candidates cannot be listed: %w", err) } - haves, err := s.st.keys(haveState) + haves, err := readAll(s.st, haveState) if err != nil { - return nil, fmt.Errorf("the nodes' haves cannot be listed: %w", err) + return nil, fmt.Errorf("the nodes' haves cannot be read: %w", err) } answers := map[string]map[string]nodeengine.CandidateAnswer{} // by proposal, then node - for _, k := range haves { + for k, v := range haves { node, _, isNode, err := nodeengineclient.ParseHaveKey(k) if err != nil || !isNode { continue } - raw, _, found, err := s.st.get(haveState, k) - if err != nil || !found { - continue - } - h, err := nodeengineclient.DecodeHaveNode(raw) + h, err := nodeengineclient.DecodeHaveNode(v.raw) if err != nil { continue } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go index 792268a6..1916b044 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go @@ -531,6 +531,14 @@ func TestTheSeatsVerbs(t *testing.T) { t.Errorf("keyed again: findings %v", findings) } }) + // A whole state is read in pages where the store has them, never a key at a time (27 s for the assignments live, + // 2026-11-14): the paged read answers what the key-by-key read answers, and asks no key alone. + direct, err := readAll(st, haveState) + paged := &pagedStore{memStore: st} + viaPages, perr := readAll(paged, haveState) + if err != nil || perr != nil || len(viaPages) != len(direct) || paged.pages == 0 || paged.gets != 0 { + t.Errorf("readAll in pages = %d keys in %d pages with %d gets (%v, %v), want the %d keys of the direct read, no get", len(viaPages), paged.pages, paged.gets, err, perr, len(direct)) + } } // wantsVerbs (novox/hq ADR 0565 §7, §8, §9): upgrade answers each module's policy and its source; a person's choice is @@ -943,3 +951,35 @@ func (f *fakeIndexer) answer(key string, body any) (json.RawMessage, bool, error raw, err := json.Marshal(a) return raw, true, err } + +// pagedStore is a memStore whose states are read in pages of one key, counting pages and single gets. +type pagedStore struct { + memStore + pages, gets int +} + +func (p *pagedStore) get(state, key string) ([]byte, uint64, bool, error) { + p.gets++ + return p.memStore.get(state, key) +} + +func (p *pagedStore) page(state, after string) (stdio.StatePage, error) { + p.pages++ + keys, err := p.memStore.keys(state) + if err != nil { + return stdio.StatePage{}, err + } + slices.Sort(keys) + for i, k := range keys { + if k <= after { + continue + } + raw, rev, _, _ := p.memStore.get(state, k) + pg := stdio.StatePage{Entries: []stdio.StateEntry{{Key: k, Value: raw, Revision: rev}}} + if i < len(keys)-1 { + pg.Next = k + } + return pg, nil + } + return stdio.StatePage{}, nil +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/store_pages.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/store_pages.go new file mode 100644 index 00000000..4e823727 --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/store_pages.go @@ -0,0 +1,69 @@ +package main + +import ( + "errors" + "fmt" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// kept is one key's value and revision, as a whole state read gives it. +type kept struct { + raw []byte + rev uint64 +} + +// pager is a store that reads a state's values many to a request (stdio.KeptState.Page): the runtime's store. +type pager interface { + page(state, after string) (stdio.StatePage, error) +} + +// pageBytes and pageBudget bound one page of a whole state read, so its answer fits one bus message and its time. +const ( + pageBytes = 768 * 1024 + pageBudget = 10 * time.Second +) + +func (runtimeStore) page(state, after string) (stdio.StatePage, error) { + return stdio.State(state).Page(after, pageBytes, pageBudget) +} + +// readAll is every key of a state with its value and revision: in pages when the store has them, else key by key. Key by +// key cost one bus request each, so the assignments (about 330 keys) took 27 s to read and a proposal outlasted its +// caller's 30 s (2026-11-14), as the indexer's view of the trunk had the hour before. +func readAll(st store, state string) (map[string]kept, error) { + out := map[string]kept{} + if p, ok := st.(pager); ok { + for after := ""; ; { + pg, err := p.page(state, after) + if errors.Is(err, errors.ErrUnsupported) { + break + } + if err != nil { + return nil, fmt.Errorf("%s cannot be read: %w", state, err) + } + for _, e := range pg.Entries { + out[e.Key] = kept{raw: e.Value, rev: e.Revision} + } + if pg.Next == "" || pg.Next == after { + return out, nil + } + after = pg.Next + } + } + keys, err := st.keys(state) + if err != nil { + return nil, fmt.Errorf("%s cannot be listed: %w", state, err) + } + for _, k := range keys { + raw, rev, found, err := st.get(state, k) + if err != nil { + return nil, fmt.Errorf("%s %s cannot be read: %w", state, k, err) + } + if found { + out[k] = kept{raw: raw, rev: rev} + } + } + return out, nil +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/wants_tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/wants_tools.go index 8608def4..2cf52c0e 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/wants_tools.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/wants_tools.go @@ -71,21 +71,15 @@ func (s *wantsTools) values(state string) (map[string][]byte, error) { return s.direct(state) } -// direct reads a state key by key, for a small state of the seat's own. +// direct reads a state whole, in pages where the runtime has them (readAll). func (s *wantsTools) direct(state string) (map[string][]byte, error) { - keys, err := s.st.keys(state) + all, err := readAll(s.st, state) if err != nil { - return nil, fmt.Errorf("%s cannot be listed: %w", state, err) + return nil, err } - out := map[string][]byte{} - for _, k := range keys { - raw, _, found, err := s.st.get(state, k) - if err != nil { - return nil, fmt.Errorf("%s %s cannot be read: %w", state, k, err) - } - if found { - out[k] = raw - } + out := make(map[string][]byte, len(all)) + for k, v := range all { + out[k] = v.raw } return out, nil }