Design 47 H1: the grants holder keeps the catalog revisions it read and learns the newest from the event (H-A-193, H-A-195) #294

Merged
mesh-tools merged 3 commits from fix/47-H1-grants-for-cache into main 2026-11-20 20:56:04 +00:00
5 changed files with 435 additions and 5 deletions
@@ -39,6 +39,7 @@ import (
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/announce"
@@ -180,8 +181,20 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
Create: creating{ctx: ctx, js: js}, Consumers: creating{ctx: ctx, js: js}, NamedPurge: creating{ctx: ctx, js: js},
Objects: sealedObjects{ctx: ctx, js: js}, Purging: ownPurging{ctx: ctx, js: js, module: account.Module}, Logf: logger.Printf,
Bus: grants.BusAddress{URL: account.URL, Fingerprint: account.Fingerprint}}
// The setting grants-for-revisions-per-branch, read at the start, and the watched branches the part follows: grants-for
// keeps those branches' revisions alone, so many to a branch (design 47 H H-A-195).
perBranch, err := grants.RevisionsPerBranchSetting(strings.TrimSpace(getenv(grants.EnvRevisionsPerBranch)))
if err != nil {
return err
}
f.Part.RevisionsPerBranch, f.Part.Followed = perBranch, branches.Followed
f.Watch = fv
f.Part.ReadingRunning(nodeengine.StreamRunning)
// The catalog-revision event names each branch's newest revision, so grants-for asks the seat mesh-catalog's revision only
// for a revision it does not hold (design 47 H H-A-193), heard on the same reader beside the branch follower.
if err := reader.AdoptEvent(meshcatalog.Seat, meshcatalog.EventCatalogRevision, f.Part.HearCatalogRevision); err != nil {
return err
}
f.HearKept = func() error { return hearRunning(ctx, account, js, f, logger.Printf) }
// The first view of the bindings (design 25 bindings-first-view): the seat mesh-broker's list, asked at each start and by
// hand, with the seat's events held until its answer is applied or its bound passes.
+135
View File
@@ -83,6 +83,8 @@ type fakeBus struct {
refuseCreate bool
// refuseUsers refuses the bus module's nats_users: the loaded list cannot be read.
refuseUsers bool
// revisionNumber, when set, numbers each revision mesh-catalog's revision answers (newest, candidate), one entry a page.
revisionNumber map[string]int64
// rootFree answers a machine's engine's root-free verb: nil is no answerer at all (no responders).
rootFree func(machines []string) (any, error)
// consumers are the durable consumers the bus holds, by stream, as nats_streams answers them for one stream; and
@@ -379,6 +381,19 @@ func (b *fakeBus) Call(key string, body any) (json.RawMessage, error) {
_ = json.Unmarshal([]byte(manifest), &m)
out.Entries = append(out.Entries, meshcatalog.CatalogEntry{Module: module, Manifest: m})
}
if n, numbered := b.revisionNumber[which]; numbered {
slices.SortFunc(out.Entries, func(x, y meshcatalog.CatalogEntry) int { return strings.Compare(x.Module, y.Module) })
out.Revision = &meshcatalog.CatalogRevision{Branch: in.Branch, Kind: "trunk", Revision: n}
if in.Group != "" {
out.Revision.Kind, out.Revision.Group, out.Revision.Heads = "candidate", in.Group, "h1"
}
for len(out.Entries) > 0 && out.Entries[0].Module <= in.After {
out.Entries = out.Entries[1:]
}
if len(out.Entries) > 1 {
out.Entries, out.Next = out.Entries[:1], out.Entries[0].Module
}
}
return json.Marshal(out)
case grants.CallStreams:
if stream, _ := args["stream"].(string); stream != "" {
@@ -5157,6 +5172,126 @@ func TestGrantsFor(t *testing.T) {
if _, err := p.GrantsFor(meshgrants.GrantsForInput{Branch: "main"}); err == nil {
t.Error("grants-for with neither a source nor a run answered")
}
// A revision never changes, so one read whole is kept: a second grants-for of the same revisions reads only the first page
// of the candidate, which names a revision kept, and none of the newest, whose number the first ask named (H-A-193: it asks
// once); once the catalog-revision events name both, a grants-for reads no page.
b = fixture()
b.revision = map[string]map[string]string{"newest": {"mod-x": xManifest, "mod-y": xManifest}, "candidate": {"mod-x": candidateX, "mod-y": xManifest}}
b.revisionNumber = map[string]int64{"newest": 9, "candidate": 8}
p = partOn(b)
pages := func() int {
b.mu.Lock()
defer b.mu.Unlock()
n := 0
for _, c := range b.calls {
if c == grants.CallRevision {
n++
}
}
return n
}
first, _ := p.GrantsFor(meshgrants.GrantsForInput{Branch: "main", Source: "7"})
read := pages()
again, _ := p.GrantsFor(meshgrants.GrantsForInput{Branch: "main", Source: "7"})
if read != 4 || pages()-read != 1 {
t.Errorf("the revisions were read in %d pages, then %d again, want 4 then 1 (the candidate's first page)", read, pages()-read)
}
if len(first.Modules) != 1 || !reflect.DeepEqual(first, again) {
t.Errorf("grants-for from kept revisions answers %+v, first %+v", again, first)
}
p = partOn(b)
for _, ev := range []string{`{"v":1,"kind":"trunk","branch":"main","revision":8,"source":7}`, `{"v":1,"kind":"trunk","branch":"main","revision":9,"source":8}`} {
if err := p.HearCatalogRevision(context.Background(), []byte(ev)); err != nil {
t.Fatal(err)
}
}
_, _ = p.GrantsFor(meshgrants.GrantsForInput{Branch: "main", Source: "7"})
read = pages()
heard, _ := p.GrantsFor(meshgrants.GrantsForInput{Branch: "main", Source: "7"})
if pages() != read || !reflect.DeepEqual(first, heard) {
t.Errorf("after the events named both revisions held, grants-for read %d pages and answered %+v, want none and %+v", pages()-read, heard, first)
}
}
// grants-for keeps what H-A-195 says of the catalog: so many revisions to a branch as the setting
// grants-for-revisions-per-branch says, a run's candidate counted within its branch's; only the followed branches', freed when
// one leaves them; and an event below the newest held is the numbering started again, which drops what is kept.
func TestGrantsForKeepsWhatItMay(t *testing.T) {
setting := func(t *testing.T, text string) int {
path := filepath.Join(t.TempDir(), "grants-for-revisions-per-branch")
if err := os.WriteFile(path, []byte(text), 0o600); err != nil {
t.Fatal(err)
}
n, err := grants.RevisionsPerBranchSetting(path)
if err != nil {
t.Fatal(err)
}
return n
}
hear := func(t *testing.T, p *grants.Part, revision, source int) {
ev := fmt.Sprintf(`{"v":1,"kind":"trunk","branch":"main","revision":%d,"source":%d}`, revision, source)
if err := p.HearCatalogRevision(context.Background(), []byte(ev)); err != nil {
t.Fatal(err)
}
}
for _, row := range []struct {
name string
// run asks the candidate by a run, else by the source revision 7; then is done after one grants-for; pages the
// revision pages the grants-for after it reads.
run bool
// perBranch is the part's bound from the start; 0 its default.
perBranch int
then func(t *testing.T, b *fakeBus, p *grants.Part, followed *[]string)
pages int
}{
{name: "the setting bounds a branch's revisions: with 2, the candidate is dropped once two newer are kept", pages: 2,
then: func(t *testing.T, b *fakeBus, p *grants.Part, _ *[]string) {
p.RevisionsPerBranch = setting(t, "2\n")
hear(t, p, 10, 0)
b.revisionNumber["newest"] = 10
_, _ = p.GrantsFor(meshgrants.GrantsForInput{Branch: "main", Source: "7"})
}},
{name: "a run's candidate counts within its branch's bound: with 1, the candidate and the newest each drop the other", run: true,
perBranch: 1, pages: 4, then: func(*testing.T, *fakeBus, *grants.Part, *[]string) {}},
{name: "a branch that leaves the watched branches frees its revisions", pages: 4,
then: func(t *testing.T, _ *fakeBus, p *grants.Part, followed *[]string) {
*followed = nil
_, _ = p.GrantsFor(meshgrants.GrantsForInput{Branch: "main", Source: "7"})
*followed = []string{"main"}
}},
{name: "an event below the newest held drops what is kept of the branch", pages: 4,
then: func(t *testing.T, _ *fakeBus, p *grants.Part, _ *[]string) { hear(t, p, 3, 0) }},
} {
t.Run(row.name, func(t *testing.T) {
b := fixture()
b.revision = map[string]map[string]string{"newest": {"mod-x": xManifest, "mod-y": xManifest}, "candidate": {"mod-x": xManifest, "mod-y": xManifest}}
b.revisionNumber = map[string]int64{"newest": 9, "candidate": 8}
followed := []string{"main"}
p := partOn(b)
p.Followed = func() []string { return followed }
p.RevisionsPerBranch = row.perBranch
in := meshgrants.GrantsForInput{Branch: "main", Source: "7"}
if row.run {
in = meshgrants.GrantsForInput{Branch: "main", Run: "r1"}
} else {
hear(t, p, 8, 7)
hear(t, p, 9, 8)
}
if got, err := p.GrantsFor(in); err != nil || got.Error != "" {
t.Fatalf("grants-for: %+v, %v", got, err)
}
row.then(t, b, p, &followed)
before := len(slices.DeleteFunc(slices.Clone(b.calls), func(c string) bool { return c != grants.CallRevision }))
_, _ = p.GrantsFor(in)
if got := len(slices.DeleteFunc(slices.Clone(b.calls), func(c string) bool { return c != grants.CallRevision })) - before; got != row.pages {
t.Errorf("grants-for read %d revision pages, want %d", got, row.pages)
}
})
}
if n, err := grants.RevisionsPerBranchSetting(filepath.Join(t.TempDir(), "none")); n != grants.DefaultRevisionsPerBranch || err != nil {
t.Errorf("with no file the setting is %d, %v; want its default", n, err)
}
}
// heldHalf is an assignment half holding rev, or none.
@@ -1,18 +1,23 @@
package grants
import (
"context"
"encoding/json"
"errors"
"fmt"
"io/fs"
"maps"
"os"
"slices"
"sort"
"strconv"
"strings"
"sync"
"git.novox.be/novox/mesh-contracts/identity-and-access/go/seats/meshgrants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// **The seat mesh-grants' read `grants-for`** (design 47 G §441, §401, A-205): the grants a candidate would compose, per
@@ -27,10 +32,240 @@ const RunGroup = "run-"
// CallRevision is the seat mesh-catalog's `revision`, as the part calls it.
const CallRevision = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbRevision
// DefaultRevisionsPerBranch is the setting grants-for-revisions-per-branch's default (design 47 H H-A-195): how many catalog
// revisions of one branch grants-for keeps, a run's candidate counted within its branch's, the oldest read dropped first;
// twice as many source revisions are kept named by the revision taken in from each.
const DefaultRevisionsPerBranch = 4
// EnvRevisionsPerBranch names the file the manifest writes the setting grants-for-revisions-per-branch into.
const EnvRevisionsPerBranch = "MESH_GRANTS_REVISIONS_PER_BRANCH_FILE"
// RevisionsPerBranchSetting reads the setting grants-for-revisions-per-branch from its file, read at the start: a missing or
// empty file is its default, one that holds no positive whole number an error naming the file.
func RevisionsPerBranchSetting(path string) (int, error) {
if path == "" {
return DefaultRevisionsPerBranch, nil
}
raw, err := os.ReadFile(path)
if errors.Is(err, fs.ErrNotExist) {
return DefaultRevisionsPerBranch, nil
}
if err != nil {
return 0, err
}
v := strings.TrimSpace(string(raw))
if v == "" {
return DefaultRevisionsPerBranch, nil
}
n, err := strconv.Atoi(v)
if err != nil || n < 1 {
return 0, fmt.Errorf("%s holds %q, which is not a positive whole number", path, v)
}
return n, nil
}
// revisionsPerBranch is the part's bound of kept revisions per branch: its setting, else the default.
func (p *Part) revisionsPerBranch() int {
if p.RevisionsPerBranch > 0 {
return p.RevisionsPerBranch
}
return DefaultRevisionsPerBranch
}
// following frees what is kept of every branch the part follows no more, and says whether branch is followed: only a followed
// branch's revisions are kept (H-A-195). With no Followed (the tools) every branch is.
func (p *Part) following(branch string) bool {
if p.Followed == nil {
return true
}
followed := map[string]bool{}
for _, b := range p.Followed() {
followed[keys.EscapeName(b)] = true
}
p.revisions.free(followed)
return followed[branch]
}
// revisionsKept keeps the catalog revisions grants-for read, whole, as manifests by module (design 47 H H-A-195): a revision
// never changes, so one read is read for good (as the runner keeps its revisionCache), a cache of the catalog seat's own truth
// and never a state of the mesh. Each is kept by the branch grants-for was asked of (escaped as a key name is), its kind,
// group, heads and number; the newest few of each followed branch are kept in memory for the process's life. It also
// holds what the catalog-revision event names (H-A-193): each branch's newest trunk revision, and the revision each source
// revision was taken in as, so grants-for asks the seat mesh-catalog's revision only for a revision it does not hold, never
// to learn the newest.
type revisionsKept struct {
mu sync.Mutex
order map[string][]string
kept map[string]map[string]catalogue.Manifest
newest map[string]int64
sources map[string]map[int64]int64
sourced map[string][]int64
}
// revisionKey names a revision of branch; every kind but a candidate's is the branch's own series.
func revisionKey(branch string, r meshcatalog.CatalogRevision) string {
if r.Kind != "candidate" {
r.Kind, r.Group, r.Heads = "trunk", "", ""
}
return strings.Join([]string{branch, r.Kind, r.Group, r.Heads, strconv.FormatInt(r.Revision, 10)}, "\x00")
}
func (k *revisionsKept) init() {
if k.kept == nil {
k.kept, k.order, k.newest = map[string]map[string]catalogue.Manifest{}, map[string][]string{}, map[string]int64{}
k.sources, k.sourced = map[string]map[int64]int64{}, map[string][]int64{}
}
}
func (k *revisionsKept) get(branch string, r meshcatalog.CatalogRevision) (map[string]catalogue.Manifest, bool) {
k.mu.Lock()
defer k.mu.Unlock()
m, ok := k.kept[revisionKey(branch, r)]
return m, ok
}
func (k *revisionsKept) put(branch string, r meshcatalog.CatalogRevision, manifests map[string]catalogue.Manifest, max int) {
k.mu.Lock()
defer k.mu.Unlock()
k.init()
key := revisionKey(branch, r)
if _, ok := k.kept[key]; !ok {
k.order[branch] = append(k.order[branch], key)
}
k.kept[key] = manifests
for len(k.order[branch]) > max {
delete(k.kept, k.order[branch][0])
k.order[branch] = k.order[branch][1:]
}
if r.Kind != "candidate" && r.Source != 0 {
k.sourceLocked(branch, r.Source, r.Revision, max)
}
}
// free drops what is kept of each branch not in followed.
func (k *revisionsKept) free(followed map[string]bool) {
k.mu.Lock()
defer k.mu.Unlock()
for _, m := range []map[string]bool{boolsOf(k.order), boolsOf(k.newest), boolsOf(k.sources)} {
for branch := range m {
if !followed[branch] {
k.dropLocked(branch)
}
}
}
}
func boolsOf[V any](m map[string]V) map[string]bool {
out := map[string]bool{}
for k := range m {
out[k] = true
}
return out
}
// dropLocked drops everything kept of branch.
func (k *revisionsKept) dropLocked(branch string) {
for _, key := range k.order[branch] {
delete(k.kept, key)
}
delete(k.order, branch)
delete(k.newest, branch)
delete(k.sources, branch)
delete(k.sourced, branch)
}
func (k *revisionsKept) sourceLocked(branch string, source, revision int64, max int) {
if k.sources[branch] == nil {
k.sources[branch] = map[int64]int64{}
}
if _, ok := k.sources[branch][source]; !ok {
k.sourced[branch] = append(k.sourced[branch], source)
}
k.sources[branch][source] = revision
for len(k.sourced[branch]) > 2*max {
delete(k.sources[branch], k.sourced[branch][0])
k.sourced[branch] = k.sourced[branch][1:]
}
}
// heard takes a trunk revision a catalog-revision event names as its branch's newest. One below the newest held is the
// numbering started again (design 47 H H-A-141): what is kept of the branch was of the earlier numbering, and is dropped.
func (k *revisionsKept) heard(branch string, revision, source int64, max int) {
k.mu.Lock()
defer k.mu.Unlock()
k.init()
if held, ok := k.newest[branch]; ok && revision < held {
k.dropLocked(branch)
}
k.newest[branch] = revision
if source != 0 {
k.sourceLocked(branch, source, revision, max)
}
}
// asked takes the newest revision an ask answered, when no event named one since the start (H-A-193: it asks once).
func (k *revisionsKept) asked(branch string, revision int64) {
k.mu.Lock()
defer k.mu.Unlock()
k.init()
if _, ok := k.newest[branch]; !ok {
k.newest[branch] = revision
}
}
// known is the revision in names that the part knows the number of without asking: the branch's newest an event named, or
// the revision a source revision was taken in as; false when it knows none.
func (k *revisionsKept) known(in meshcatalog.RevisionInput) (int64, bool) {
k.mu.Lock()
defer k.mu.Unlock()
switch {
case in.Group != "" || in.Heads != "" || in.Revision != 0:
return 0, false
case in.Source != 0:
n, ok := k.sources[in.Branch][in.Source]
return n, ok
default:
n, ok := k.newest[in.Branch]
return n, ok
}
}
// HearCatalogRevision takes one catalog-revision event of the seat mesh-catalog, as the part's process hears it (design 47 H
// H-A-193): a trunk revision is its branch's newest, and the revision its source revision was taken in as; a candidate's moves
// nothing here. Data that is no such event is an error.
func (p *Part) HearCatalogRevision(_ context.Context, data []byte) error {
var ev meshcatalog.CatalogRevisionEvent
if err := json.Unmarshal(data, &ev); err != nil || ev.Revision <= 0 {
return fmt.Errorf("not a %s event: %.200s", meshcatalog.EventCatalogRevision, data)
}
if ev.Kind == "candidate" || !p.following(ev.Branch) {
return nil
}
p.revisions.heard(ev.Branch, ev.Revision, ev.Source, p.revisionsPerBranch())
return nil
}
// revisionEntries reads a catalog graph revision's entries, page after page, as manifests by module. none says the branch has
// no revision yet.
// no revision yet. A revision whose number the part knows (the branch's newest an event named, a source revision's) and holds
// is answered from what it holds with no read; one it knows but does not hold is read by that number. Otherwise a revision
// read whole is kept: once the first page of an answer names a revision that is kept, it is answered from what is kept,
// without the pages after it. The branch's newest is read on by the number its first page names, so its pages are of one
// revision though a newer one is written between them; a revision whose pages name another is not kept.
func (p *Part) revisionEntries(in meshcatalog.RevisionInput) (manifests map[string]catalogue.Manifest, none bool, err error) {
branch := in.Branch
followed := p.following(branch)
newestAsked := in.Group == "" && in.Heads == "" && in.Source == 0 && in.Revision == 0
if n, ok := p.revisions.known(in); ok {
if m, held := p.revisions.get(branch, meshcatalog.CatalogRevision{Revision: n}); held {
return m, false, nil
}
if newestAsked {
in.Revision = n
}
}
manifests = map[string]catalogue.Manifest{}
var first *meshcatalog.CatalogRevision
keep := true
for {
raw, err := p.Call.Call(CallRevision, in)
if err != nil {
@@ -43,6 +278,23 @@ func (p *Part) revisionEntries(in meshcatalog.RevisionInput) (manifests map[stri
if a.NoneYet {
return nil, true, nil
}
switch {
case a.Revision == nil:
keep = false
case first == nil:
if newestAsked && followed {
p.revisions.asked(branch, a.Revision.Revision)
}
if m, ok := p.revisions.get(branch, *a.Revision); ok {
return m, false, nil
}
first = a.Revision
if newestAsked {
in.Revision = a.Revision.Revision
}
case revisionKey(branch, *a.Revision) != revisionKey(branch, *first):
keep = false
}
for _, e := range a.Entries {
body, err := json.Marshal(e.Manifest)
if err != nil || len(e.Manifest) == 0 {
@@ -55,6 +307,9 @@ func (p *Part) revisionEntries(in meshcatalog.RevisionInput) (manifests map[stri
manifests[e.Module] = m
}
if a.Next == "" || a.Next == in.After {
if keep && first != nil && followed {
p.revisions.put(branch, *first, manifests, p.revisionsPerBranch())
}
return manifests, false, nil
}
in.After = a.Next
+11 -3
View File
@@ -194,11 +194,19 @@ type Part struct {
// SeatList is the seat list reader, which gates a module whose seats the list has not named (inputs.go, SeatGate); nil
// gates none.
SeatList SeatGate
// RevisionsPerBranch is the setting grants-for-revisions-per-branch, read at the start (grantsfor.go, design 47 H H-A-195);
// 0 is its default.
RevisionsPerBranch int
// Followed is the watched branches the part follows, as named (the branch follower's): grants-for keeps the revisions
// of these alone, and frees a branch's once it leaves them; nil in the tools, where every branch is kept.
Followed func() []string
cache Manifests
mu sync.Mutex
last Result
made creations
// revisions are the catalog revisions grants-for read, kept whole by their number (grantsfor.go).
revisions revisionsKept
mu sync.Mutex
last Result
made creations
// askSeen is the revision of the last ask to compose (KeyAsked) this part acted on.
askSeen uint64
// namedPurged says the last pass found the build seats' named asks unused and purged them, so the next purges only
+20 -1
View File
@@ -64,6 +64,16 @@
"seat:mesh-broker.list",
"seat:mesh-recovery.kept-points"
],
"config": {
"grants-for-revisions-per-branch": {
"type": "number",
"description": "how many catalog revisions of one watched branch the grants part keeps once read for grants-for, a run's candidate counted within its branch's, the oldest dropped first; each some 1.4 MB (design 47 H H-A-195).",
"minimum": 1,
"maximum": 32,
"trusted": false,
"default": 4
}
},
"emits": [
"asserted",
"grant-written",
@@ -156,6 +166,14 @@
"place": ".",
"unknown": "program"
},
{
"id": "grants-for-revisions-per-branch",
"type": "file",
"path": "${dir:state}/grants-for-revisions-per-branch",
"owner": "mesh-grants",
"mode": "0600",
"content": "${config:grants-for-revisions-per-branch}"
},
{
"id": "grants",
"type": "process",
@@ -165,7 +183,8 @@
"./mesh-grants"
],
"env": {
"MESH_BROKER_FILE": "${dir:state}/broker"
"MESH_BROKER_FILE": "${dir:state}/broker",
"MESH_GRANTS_REVISIONS_PER_BRANCH_FILE": "${dir:state}/grants-for-revisions-per-branch"
},
"user": "mesh-grants",
"health": {