mesh-grants: grants-for keeps the catalog revisions it read

grants-for read two whole catalog revisions page by page on every call
(about 1.4 MB each) and answered in 3.4 to 5.0 s, so the pipeline
runner's 5 s read bound cut it and every widening touching the vault
said "could not run". A revision never changes, so the part now keeps
each revision it read whole, by branch, kind, group, heads and number,
the newest four per branch for the process's life (as the runner keeps
its revisionCache); once a first page names a kept revision the pages
after it are not read. The branch's newest is read on by the number its
first page names, and a revision whose pages name another is not kept.

Quick fix (not chosen): raise the runner's widens read bound, a catalog
merge that hides the cost. Thorough fix (chosen): the cache here. The
compose of every account twice is kept whole: narrowing it to the
candidate's modules would stop a refused principal elsewhere from
failing grants-for, which is not obviously safe.

Builds: design 47 G §441, §401; H §774
This commit is contained in:
2026-11-20 21:40:15 +01:00
parent e4781edcd6
commit 20d59ff4f0
3 changed files with 107 additions and 4 deletions
+39
View File
@@ -83,6 +83,8 @@ type fakeBus struct {
refuseCreate bool
// refuseUsers refuses the bus module's nats_users: the loaded list cannot be read.
refuseUsers bool
// revisionNumber, when set, numbers each revision mesh-catalog's revision answers (newest, candidate), one entry a page.
revisionNumber map[string]int64
// rootFree answers a machine's engine's root-free verb: nil is no answerer at all (no responders).
rootFree func(machines []string) (any, error)
// consumers are the durable consumers the bus holds, by stream, as nats_streams answers them for one stream; and
@@ -379,6 +381,16 @@ func (b *fakeBus) Call(key string, body any) (json.RawMessage, error) {
_ = json.Unmarshal([]byte(manifest), &m)
out.Entries = append(out.Entries, meshcatalog.CatalogEntry{Module: module, Manifest: m})
}
if n, numbered := b.revisionNumber[which]; numbered {
slices.SortFunc(out.Entries, func(x, y meshcatalog.CatalogEntry) int { return strings.Compare(x.Module, y.Module) })
out.Revision = &meshcatalog.CatalogRevision{Branch: in.Branch, Kind: "trunk", Revision: n}
for len(out.Entries) > 0 && out.Entries[0].Module <= in.After {
out.Entries = out.Entries[1:]
}
if len(out.Entries) > 1 {
out.Entries, out.Next = out.Entries[:1], out.Entries[0].Module
}
}
return json.Marshal(out)
case grants.CallStreams:
if stream, _ := args["stream"].(string); stream != "" {
@@ -5157,6 +5169,33 @@ func TestGrantsFor(t *testing.T) {
if _, err := p.GrantsFor(meshgrants.GrantsForInput{Branch: "main"}); err == nil {
t.Error("grants-for with neither a source nor a run answered")
}
// A revision never changes, so one read whole is kept: a second grants-for of the same revisions reads only the first page
// of each, which names a revision kept, and answers the same.
b = fixture()
b.revision = map[string]map[string]string{"newest": {"mod-x": xManifest, "mod-y": xManifest}, "candidate": {"mod-x": candidateX, "mod-y": xManifest}}
b.revisionNumber = map[string]int64{"newest": 9, "candidate": 8}
p = partOn(b)
pages := func() int {
b.mu.Lock()
defer b.mu.Unlock()
n := 0
for _, c := range b.calls {
if c == grants.CallRevision {
n++
}
}
return n
}
first, _ := p.GrantsFor(meshgrants.GrantsForInput{Branch: "main", Source: "7"})
read := pages()
again, _ := p.GrantsFor(meshgrants.GrantsForInput{Branch: "main", Source: "7"})
if read != 4 || pages()-read != 2 {
t.Errorf("the revisions were read in %d pages, then %d again, want 4 then 2 (the first page of each)", read, pages()-read)
}
if len(first.Modules) != 1 || !reflect.DeepEqual(first, again) {
t.Errorf("grants-for from kept revisions answers %+v, first %+v", again, first)
}
}
// heldHalf is an assignment half holding rev, or none.
@@ -9,6 +9,7 @@ import (
"sort"
"strconv"
"strings"
"sync"
"git.novox.be/novox/mesh-contracts/identity-and-access/go/seats/meshgrants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
@@ -27,10 +28,54 @@ const RunGroup = "run-"
// CallRevision is the seat mesh-catalog's `revision`, as the part calls it.
const CallRevision = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbRevision
// revisionsKeptPerBranch is how many catalog revisions of one branch grants-for keeps, the oldest read dropped first.
const revisionsKeptPerBranch = 4
// revisionsKept keeps the catalog revisions grants-for read, whole, as manifests by module: a revision never changes, so one
// read is read for good (as the runner keeps its revisionCache). Each is kept by its branch, kind, group, heads and number,
// as the first page of its answer names them; the newest few of each branch are kept for the process's life.
type revisionsKept struct {
mu sync.Mutex
order map[string][]string
kept map[string]map[string]catalogue.Manifest
}
func revisionKey(r meshcatalog.CatalogRevision) string {
return strings.Join([]string{r.Branch, r.Kind, r.Group, r.Heads, strconv.FormatInt(r.Revision, 10)}, "\x00")
}
func (k *revisionsKept) get(r meshcatalog.CatalogRevision) (map[string]catalogue.Manifest, bool) {
k.mu.Lock()
defer k.mu.Unlock()
m, ok := k.kept[revisionKey(r)]
return m, ok
}
func (k *revisionsKept) put(r meshcatalog.CatalogRevision, manifests map[string]catalogue.Manifest) {
k.mu.Lock()
defer k.mu.Unlock()
if k.kept == nil {
k.kept, k.order = map[string]map[string]catalogue.Manifest{}, map[string][]string{}
}
key := revisionKey(r)
if _, ok := k.kept[key]; !ok {
k.order[r.Branch] = append(k.order[r.Branch], key)
}
k.kept[key] = manifests
for len(k.order[r.Branch]) > revisionsKeptPerBranch {
delete(k.kept, k.order[r.Branch][0])
k.order[r.Branch] = k.order[r.Branch][1:]
}
}
// revisionEntries reads a catalog graph revision's entries, page after page, as manifests by module. none says the branch has
// no revision yet.
// no revision yet. A revision read whole is kept: once the first page of an answer names a revision that is kept, it is
// answered from what is kept, without the pages after it. The branch's newest is read on by the number its first page names, so
// its pages are of one revision though a newer one is written between them; a revision whose pages name another is not kept.
func (p *Part) revisionEntries(in meshcatalog.RevisionInput) (manifests map[string]catalogue.Manifest, none bool, err error) {
manifests = map[string]catalogue.Manifest{}
var first *meshcatalog.CatalogRevision
keep := true
for {
raw, err := p.Call.Call(CallRevision, in)
if err != nil {
@@ -43,6 +88,20 @@ func (p *Part) revisionEntries(in meshcatalog.RevisionInput) (manifests map[stri
if a.NoneYet {
return nil, true, nil
}
switch {
case a.Revision == nil:
keep = false
case first == nil:
if m, ok := p.revisions.get(*a.Revision); ok {
return m, false, nil
}
first = a.Revision
if in.Group == "" && in.Heads == "" && in.Source == 0 && in.Revision == 0 {
in.Revision = a.Revision.Revision
}
case revisionKey(*a.Revision) != revisionKey(*first):
keep = false
}
for _, e := range a.Entries {
body, err := json.Marshal(e.Manifest)
if err != nil || len(e.Manifest) == 0 {
@@ -55,6 +114,9 @@ func (p *Part) revisionEntries(in meshcatalog.RevisionInput) (manifests map[stri
manifests[e.Module] = m
}
if a.Next == "" || a.Next == in.After {
if keep && first != nil {
p.revisions.put(*first, manifests)
}
return manifests, false, nil
}
in.After = a.Next
+5 -3
View File
@@ -196,9 +196,11 @@ type Part struct {
SeatList SeatGate
cache Manifests
mu sync.Mutex
last Result
made creations
// revisions are the catalog revisions grants-for read, kept whole by their number (grantsfor.go).
revisions revisionsKept
mu sync.Mutex
last Result
made creations
// askSeen is the revision of the last ask to compose (KeyAsked) this part acted on.
askSeen uint64
// namedPurged says the last pass found the build seats' named asks unused and purged them, so the next purges only