diff --git a/modules/mesh-grants/grants_test.go b/modules/mesh-grants/grants_test.go index b4cbb5f4..a0313a5d 100644 --- a/modules/mesh-grants/grants_test.go +++ b/modules/mesh-grants/grants_test.go @@ -83,6 +83,8 @@ type fakeBus struct { refuseCreate bool // refuseUsers refuses the bus module's nats_users: the loaded list cannot be read. refuseUsers bool + // revisionNumber, when set, numbers each revision mesh-catalog's revision answers (newest, candidate), one entry a page. + revisionNumber map[string]int64 // rootFree answers a machine's engine's root-free verb: nil is no answerer at all (no responders). rootFree func(machines []string) (any, error) // consumers are the durable consumers the bus holds, by stream, as nats_streams answers them for one stream; and @@ -379,6 +381,16 @@ func (b *fakeBus) Call(key string, body any) (json.RawMessage, error) { _ = json.Unmarshal([]byte(manifest), &m) out.Entries = append(out.Entries, meshcatalog.CatalogEntry{Module: module, Manifest: m}) } + if n, numbered := b.revisionNumber[which]; numbered { + slices.SortFunc(out.Entries, func(x, y meshcatalog.CatalogEntry) int { return strings.Compare(x.Module, y.Module) }) + out.Revision = &meshcatalog.CatalogRevision{Branch: in.Branch, Kind: "trunk", Revision: n} + for len(out.Entries) > 0 && out.Entries[0].Module <= in.After { + out.Entries = out.Entries[1:] + } + if len(out.Entries) > 1 { + out.Entries, out.Next = out.Entries[:1], out.Entries[0].Module + } + } return json.Marshal(out) case grants.CallStreams: if stream, _ := args["stream"].(string); stream != "" { @@ -5157,6 +5169,33 @@ func TestGrantsFor(t *testing.T) { if _, err := p.GrantsFor(meshgrants.GrantsForInput{Branch: "main"}); err == nil { t.Error("grants-for with neither a source nor a run answered") } + + // A revision never changes, so one read whole is kept: a second grants-for of the same revisions reads only the first page + // of each, which names a revision kept, and answers the same. + b = fixture() + b.revision = map[string]map[string]string{"newest": {"mod-x": xManifest, "mod-y": xManifest}, "candidate": {"mod-x": candidateX, "mod-y": xManifest}} + b.revisionNumber = map[string]int64{"newest": 9, "candidate": 8} + p = partOn(b) + pages := func() int { + b.mu.Lock() + defer b.mu.Unlock() + n := 0 + for _, c := range b.calls { + if c == grants.CallRevision { + n++ + } + } + return n + } + first, _ := p.GrantsFor(meshgrants.GrantsForInput{Branch: "main", Source: "7"}) + read := pages() + again, _ := p.GrantsFor(meshgrants.GrantsForInput{Branch: "main", Source: "7"}) + if read != 4 || pages()-read != 2 { + t.Errorf("the revisions were read in %d pages, then %d again, want 4 then 2 (the first page of each)", read, pages()-read) + } + if len(first.Modules) != 1 || !reflect.DeepEqual(first, again) { + t.Errorf("grants-for from kept revisions answers %+v, first %+v", again, first) + } } // heldHalf is an assignment half holding rev, or none. diff --git a/modules/mesh-grants/internal/grants/grantsfor.go b/modules/mesh-grants/internal/grants/grantsfor.go index 81985a6c..43b34566 100644 --- a/modules/mesh-grants/internal/grants/grantsfor.go +++ b/modules/mesh-grants/internal/grants/grantsfor.go @@ -9,6 +9,7 @@ import ( "sort" "strconv" "strings" + "sync" "git.novox.be/novox/mesh-contracts/identity-and-access/go/seats/meshgrants" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" @@ -27,10 +28,54 @@ const RunGroup = "run-" // CallRevision is the seat mesh-catalog's `revision`, as the part calls it. const CallRevision = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbRevision +// revisionsKeptPerBranch is how many catalog revisions of one branch grants-for keeps, the oldest read dropped first. +const revisionsKeptPerBranch = 4 + +// revisionsKept keeps the catalog revisions grants-for read, whole, as manifests by module: a revision never changes, so one +// read is read for good (as the runner keeps its revisionCache). Each is kept by its branch, kind, group, heads and number, +// as the first page of its answer names them; the newest few of each branch are kept for the process's life. +type revisionsKept struct { + mu sync.Mutex + order map[string][]string + kept map[string]map[string]catalogue.Manifest +} + +func revisionKey(r meshcatalog.CatalogRevision) string { + return strings.Join([]string{r.Branch, r.Kind, r.Group, r.Heads, strconv.FormatInt(r.Revision, 10)}, "\x00") +} + +func (k *revisionsKept) get(r meshcatalog.CatalogRevision) (map[string]catalogue.Manifest, bool) { + k.mu.Lock() + defer k.mu.Unlock() + m, ok := k.kept[revisionKey(r)] + return m, ok +} + +func (k *revisionsKept) put(r meshcatalog.CatalogRevision, manifests map[string]catalogue.Manifest) { + k.mu.Lock() + defer k.mu.Unlock() + if k.kept == nil { + k.kept, k.order = map[string]map[string]catalogue.Manifest{}, map[string][]string{} + } + key := revisionKey(r) + if _, ok := k.kept[key]; !ok { + k.order[r.Branch] = append(k.order[r.Branch], key) + } + k.kept[key] = manifests + for len(k.order[r.Branch]) > revisionsKeptPerBranch { + delete(k.kept, k.order[r.Branch][0]) + k.order[r.Branch] = k.order[r.Branch][1:] + } +} + // revisionEntries reads a catalog graph revision's entries, page after page, as manifests by module. none says the branch has -// no revision yet. +// no revision yet. A revision read whole is kept: once the first page of an answer names a revision that is kept, it is +// answered from what is kept, without the pages after it. The branch's newest is read on by the number its first page names, so +// its pages are of one revision though a newer one is written between them; a revision whose pages name another is not kept. func (p *Part) revisionEntries(in meshcatalog.RevisionInput) (manifests map[string]catalogue.Manifest, none bool, err error) { manifests = map[string]catalogue.Manifest{} + var first *meshcatalog.CatalogRevision + keep := true for { raw, err := p.Call.Call(CallRevision, in) if err != nil { @@ -43,6 +88,20 @@ func (p *Part) revisionEntries(in meshcatalog.RevisionInput) (manifests map[stri if a.NoneYet { return nil, true, nil } + switch { + case a.Revision == nil: + keep = false + case first == nil: + if m, ok := p.revisions.get(*a.Revision); ok { + return m, false, nil + } + first = a.Revision + if in.Group == "" && in.Heads == "" && in.Source == 0 && in.Revision == 0 { + in.Revision = a.Revision.Revision + } + case revisionKey(*a.Revision) != revisionKey(*first): + keep = false + } for _, e := range a.Entries { body, err := json.Marshal(e.Manifest) if err != nil || len(e.Manifest) == 0 { @@ -55,6 +114,9 @@ func (p *Part) revisionEntries(in meshcatalog.RevisionInput) (manifests map[stri manifests[e.Module] = m } if a.Next == "" || a.Next == in.After { + if keep && first != nil { + p.revisions.put(*first, manifests) + } return manifests, false, nil } in.After = a.Next diff --git a/modules/mesh-grants/internal/grants/part.go b/modules/mesh-grants/internal/grants/part.go index 2ee46a4f..744f40a7 100644 --- a/modules/mesh-grants/internal/grants/part.go +++ b/modules/mesh-grants/internal/grants/part.go @@ -196,9 +196,11 @@ type Part struct { SeatList SeatGate cache Manifests - mu sync.Mutex - last Result - made creations + // revisions are the catalog revisions grants-for read, kept whole by their number (grantsfor.go). + revisions revisionsKept + mu sync.Mutex + last Result + made creations // askSeen is the revision of the last ask to compose (KeyAsked) this part acted on. askSeen uint64 // namedPurged says the last pass found the build seats' named asks unused and purged them, so the next purges only