diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index 664447b7..8990f1e5 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -285,7 +285,9 @@ func main() { acts := meshhandactsclient.Recorder{Ask: stdio.Ask} tools := Tools(asks, time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState)), acts) // The modules and the module graph are the newest catalog graph revision's (milestone 47/G): revision.go. - revisions := bundleObjects{busstate.ObjectsOf(index.CatalogRevisionsObjects)} + revisions := index.KeepEntries(bundleObjects{busstate.ObjectsOf(index.CatalogRevisionsObjects)}, index.EntryKeptBytes, func(format string, a ...any) { + fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] "+format+"\n", a...) + }) trunk := index.TrunkStates{Objects: revisions, Under: keptSource, Kept: &index.KeptTrunk{Say: func(format string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] "+format+"\n", a...) }}} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index b99d4b94..23d65f0a 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -356,7 +356,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // The view of the trunk is the newest trunk revision of the catalog graph the objects keep (milestone 47/G, ADR 0656 §6: // one form of the fact), read once here and kept as a revision after every write; objects not on the bus yet, or none // kept, start it empty, and the first start's catch-up takes the trunk in once the gate's source is read (catchUp). - revisions := newBusObjects(js, cred, index.CatalogRevisionsObjects) + revisions := index.KeepEntries(newBusObjects(js, cred, index.CatalogRevisionsObjects), index.EntryKeptBytes, logger.Printf) modules, graph, at, err := index.TrunkView(ctx, revisions) switch { case errors.Is(err, process.ErrNotYet): diff --git a/modules/mesh-delivery-indexer/internal/index/entrykeeper.go b/modules/mesh-delivery-indexer/internal/index/entrykeeper.go new file mode 100644 index 00000000..5b36712a --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/entrykeeper.go @@ -0,0 +1,113 @@ +package index + +import ( + "context" + "strings" + "sync" +) + +// EntryKeptBytes bounds the bytes of entry objects the indexer holds in memory. +const EntryKeptBytes = 64 << 20 + +// entryKeeper is the catalog graph's objects with every entry object (entry.) kept in memory once read or written: +// an entry is written once and never changes, and every revision that lists it shares it, so the indexer reads each entry +// from the bus once per process (design 47 G §354, A-252). Revision lists and every other name go to the bus as before. A +// Delete drops the entry, so a revision that still lists it fails naming it, as before. +type entryKeeper struct { + Objects + limit int64 + logf func(string, ...any) + + mu sync.Mutex + kept map[string][]byte + size int64 + // Hits and Reads count the entry Gets answered from memory and from the bus. + hits, reads int64 +} + +func newEntryKeeper(inner Objects, limit int64, logf func(string, ...any)) *entryKeeper { + return &entryKeeper{Objects: inner, limit: limit, logf: logf, kept: map[string][]byte{}} +} + +func isEntry(name string) bool { return strings.HasPrefix(name, "entry.") } + +// keep holds data under name; over the bound it drops everything kept first and says so once per drop. +func (k *entryKeeper) keep(name string, data []byte) { + k.mu.Lock() + defer k.mu.Unlock() + if old, ok := k.kept[name]; ok { + k.size -= int64(len(old)) + delete(k.kept, name) + } + if int64(len(data)) > k.limit { + return + } + if k.size+int64(len(data)) > k.limit { + k.logf("the %d entry objects kept in memory (%d bytes) pass the bound of %d bytes: dropped, they are read from the bus again", len(k.kept), k.size, k.limit) + k.kept, k.size = map[string][]byte{}, 0 + } + k.kept[name] = data + k.size += int64(len(data)) +} + +func (k *entryKeeper) drop(name string) { + k.mu.Lock() + defer k.mu.Unlock() + if old, ok := k.kept[name]; ok { + k.size -= int64(len(old)) + delete(k.kept, name) + } +} + +func (k *entryKeeper) Get(ctx context.Context, name string) ([]byte, error) { + if !isEntry(name) { + return k.Objects.Get(ctx, name) + } + k.mu.Lock() + data, ok := k.kept[name] + if ok { + k.hits++ + } + k.mu.Unlock() + if ok { + return data, nil + } + data, err := k.Objects.Get(ctx, name) + if err != nil { + return nil, err + } + k.mu.Lock() + k.reads++ + k.mu.Unlock() + k.keep(name, data) + return data, nil +} + +func (k *entryKeeper) Put(ctx context.Context, name string, data []byte) error { + if err := k.Objects.Put(ctx, name, data); err != nil { + if isEntry(name) { + k.drop(name) + } + return err + } + if isEntry(name) { + k.keep(name, append([]byte(nil), data...)) + } + return nil +} + +func (k *entryKeeper) Delete(ctx context.Context, name string) error { + if isEntry(name) { + k.drop(name) + } + err := k.Objects.Delete(ctx, name) + if isEntry(name) { + k.drop(name) + } + return err +} + +// KeepEntries is inner with its entry objects kept in memory (see entryKeeper), up to limit bytes; logf says a drop. +func KeepEntries(inner Objects, limit int64, logf func(string, ...any)) Objects { + return newEntryKeeper(inner, limit, logf) +} diff --git a/modules/mesh-delivery-indexer/internal/index/entrykeeper_test.go b/modules/mesh-delivery-indexer/internal/index/entrykeeper_test.go new file mode 100644 index 00000000..d4132da2 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/entrykeeper_test.go @@ -0,0 +1,80 @@ +package index + +import ( + "context" + "errors" + "testing" +) + +type countingObjects struct { + Objects + data map[string][]byte + gets int +} + +func (c *countingObjects) Put(_ context.Context, n string, d []byte) error { c.data[n] = d; return nil } +func (c *countingObjects) Get(_ context.Context, n string) ([]byte, error) { + c.gets++ + d, ok := c.data[n] + if !ok { + return nil, ErrNoObject + } + return d, nil +} +func (c *countingObjects) Delete(_ context.Context, n string) error { delete(c.data, n); return nil } + +func TestEntryKeeper(t *testing.T) { + ctx := context.Background() + for _, tc := range []struct { + name string + limit int64 + run func(t *testing.T, k *entryKeeper, in *countingObjects) + wantGets int + wantDrops int + }{ + {"second Get of an entry is from memory", 1000, func(t *testing.T, k *entryKeeper, in *countingObjects) { + in.data["entry.a"] = []byte("A") + for i := 0; i < 3; i++ { + if d, err := k.Get(ctx, "entry.a"); err != nil || string(d) != "A" { + t.Fatalf("get: %q %v", d, err) + } + } + }, 1, 0}, + {"a Put entry is kept, never read", 1000, func(t *testing.T, k *entryKeeper, in *countingObjects) { + if err := k.Put(ctx, "entry.a", []byte("A")); err != nil { + t.Fatal(err) + } + if d, _ := k.Get(ctx, "entry.a"); string(d) != "A" { + t.Fatalf("got %q", d) + } + }, 0, 0}, + {"Delete drops it and the next Get is an error naming no object", 1000, func(t *testing.T, k *entryKeeper, in *countingObjects) { + _ = k.Put(ctx, "entry.a", []byte("A")) + _ = k.Delete(ctx, "entry.a") + if _, err := k.Get(ctx, "entry.a"); !errors.Is(err, ErrNoObject) { + t.Fatalf("err %v", err) + } + }, 1, 0}, + {"a name that is no entry always reaches the bus", 1000, func(t *testing.T, k *entryKeeper, in *countingObjects) { + in.data["trunk.1"] = []byte("L") + _, _ = k.Get(ctx, "trunk.1") + _, _ = k.Get(ctx, "trunk.1") + }, 2, 0}, + {"over the bound everything kept is dropped and said", 3, func(t *testing.T, k *entryKeeper, in *countingObjects) { + in.data["entry.a"], in.data["entry.b"] = []byte("AA"), []byte("BB") + _, _ = k.Get(ctx, "entry.a") + _, _ = k.Get(ctx, "entry.b") // drops a + _, _ = k.Get(ctx, "entry.a") // from the bus again + }, 3, 2}, + } { + t.Run(tc.name, func(t *testing.T) { + in := &countingObjects{data: map[string][]byte{}} + drops := 0 + k := newEntryKeeper(in, tc.limit, func(string, ...any) { drops++ }) + tc.run(t, k, in) + if in.gets != tc.wantGets || drops != tc.wantDrops { + t.Fatalf("bus gets %d (want %d), drops said %d (want %d)", in.gets, tc.wantGets, drops, tc.wantDrops) + } + }) + } +}