indexer: the indexer reads each catalog graph entry from the bus once per process
An entry object entry.<digest> is written once and never changes, yet every revision read (the verb revision on each page, the composition steps, about 20 callers of ReadRevision) took one object Get per entry, each bounded by the bus client's deadline: composition steps ended "could not run: the step's catalog revision is not read ... context deadline exceeded". The indexer now keeps the bytes of entry objects it has read or written in memory (bound 64 MiB, dropped whole and logged once when passed); a Delete drops the entry, so a revision that still lists it fails naming it. Revision lists are read from the bus as before. Builds: design 47 G §354 (composition reads the revision), A-252
This commit is contained in:
@@ -285,7 +285,9 @@ func main() {
|
||||
acts := meshhandactsclient.Recorder{Ask: stdio.Ask}
|
||||
tools := Tools(asks, time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState)), acts)
|
||||
// The modules and the module graph are the newest catalog graph revision's (milestone 47/G): revision.go.
|
||||
revisions := bundleObjects{busstate.ObjectsOf(index.CatalogRevisionsObjects)}
|
||||
revisions := index.KeepEntries(bundleObjects{busstate.ObjectsOf(index.CatalogRevisionsObjects)}, index.EntryKeptBytes, func(format string, a ...any) {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] "+format+"\n", a...)
|
||||
})
|
||||
trunk := index.TrunkStates{Objects: revisions, Under: keptSource, Kept: &index.KeptTrunk{Say: func(format string, a ...any) {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] "+format+"\n", a...)
|
||||
}}}
|
||||
|
||||
@@ -356,7 +356,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// The view of the trunk is the newest trunk revision of the catalog graph the objects keep (milestone 47/G, ADR 0656 §6:
|
||||
// one form of the fact), read once here and kept as a revision after every write; objects not on the bus yet, or none
|
||||
// kept, start it empty, and the first start's catch-up takes the trunk in once the gate's source is read (catchUp).
|
||||
revisions := newBusObjects(js, cred, index.CatalogRevisionsObjects)
|
||||
revisions := index.KeepEntries(newBusObjects(js, cred, index.CatalogRevisionsObjects), index.EntryKeptBytes, logger.Printf)
|
||||
modules, graph, at, err := index.TrunkView(ctx, revisions)
|
||||
switch {
|
||||
case errors.Is(err, process.ErrNotYet):
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// EntryKeptBytes bounds the bytes of entry objects the indexer holds in memory.
|
||||
const EntryKeptBytes = 64 << 20
|
||||
|
||||
// entryKeeper is the catalog graph's objects with every entry object (entry.<digest>) kept in memory once read or written:
|
||||
// an entry is written once and never changes, and every revision that lists it shares it, so the indexer reads each entry
|
||||
// from the bus once per process (design 47 G §354, A-252). Revision lists and every other name go to the bus as before. A
|
||||
// Delete drops the entry, so a revision that still lists it fails naming it, as before.
|
||||
type entryKeeper struct {
|
||||
Objects
|
||||
limit int64
|
||||
logf func(string, ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
kept map[string][]byte
|
||||
size int64
|
||||
// Hits and Reads count the entry Gets answered from memory and from the bus.
|
||||
hits, reads int64
|
||||
}
|
||||
|
||||
func newEntryKeeper(inner Objects, limit int64, logf func(string, ...any)) *entryKeeper {
|
||||
return &entryKeeper{Objects: inner, limit: limit, logf: logf, kept: map[string][]byte{}}
|
||||
}
|
||||
|
||||
func isEntry(name string) bool { return strings.HasPrefix(name, "entry.") }
|
||||
|
||||
// keep holds data under name; over the bound it drops everything kept first and says so once per drop.
|
||||
func (k *entryKeeper) keep(name string, data []byte) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
if old, ok := k.kept[name]; ok {
|
||||
k.size -= int64(len(old))
|
||||
delete(k.kept, name)
|
||||
}
|
||||
if int64(len(data)) > k.limit {
|
||||
return
|
||||
}
|
||||
if k.size+int64(len(data)) > k.limit {
|
||||
k.logf("the %d entry objects kept in memory (%d bytes) pass the bound of %d bytes: dropped, they are read from the bus again", len(k.kept), k.size, k.limit)
|
||||
k.kept, k.size = map[string][]byte{}, 0
|
||||
}
|
||||
k.kept[name] = data
|
||||
k.size += int64(len(data))
|
||||
}
|
||||
|
||||
func (k *entryKeeper) drop(name string) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
if old, ok := k.kept[name]; ok {
|
||||
k.size -= int64(len(old))
|
||||
delete(k.kept, name)
|
||||
}
|
||||
}
|
||||
|
||||
func (k *entryKeeper) Get(ctx context.Context, name string) ([]byte, error) {
|
||||
if !isEntry(name) {
|
||||
return k.Objects.Get(ctx, name)
|
||||
}
|
||||
k.mu.Lock()
|
||||
data, ok := k.kept[name]
|
||||
if ok {
|
||||
k.hits++
|
||||
}
|
||||
k.mu.Unlock()
|
||||
if ok {
|
||||
return data, nil
|
||||
}
|
||||
data, err := k.Objects.Get(ctx, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
k.mu.Lock()
|
||||
k.reads++
|
||||
k.mu.Unlock()
|
||||
k.keep(name, data)
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func (k *entryKeeper) Put(ctx context.Context, name string, data []byte) error {
|
||||
if err := k.Objects.Put(ctx, name, data); err != nil {
|
||||
if isEntry(name) {
|
||||
k.drop(name)
|
||||
}
|
||||
return err
|
||||
}
|
||||
if isEntry(name) {
|
||||
k.keep(name, append([]byte(nil), data...))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (k *entryKeeper) Delete(ctx context.Context, name string) error {
|
||||
if isEntry(name) {
|
||||
k.drop(name)
|
||||
}
|
||||
err := k.Objects.Delete(ctx, name)
|
||||
if isEntry(name) {
|
||||
k.drop(name)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// KeepEntries is inner with its entry objects kept in memory (see entryKeeper), up to limit bytes; logf says a drop.
|
||||
func KeepEntries(inner Objects, limit int64, logf func(string, ...any)) Objects {
|
||||
return newEntryKeeper(inner, limit, logf)
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type countingObjects struct {
|
||||
Objects
|
||||
data map[string][]byte
|
||||
gets int
|
||||
}
|
||||
|
||||
func (c *countingObjects) Put(_ context.Context, n string, d []byte) error { c.data[n] = d; return nil }
|
||||
func (c *countingObjects) Get(_ context.Context, n string) ([]byte, error) {
|
||||
c.gets++
|
||||
d, ok := c.data[n]
|
||||
if !ok {
|
||||
return nil, ErrNoObject
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
func (c *countingObjects) Delete(_ context.Context, n string) error { delete(c.data, n); return nil }
|
||||
|
||||
func TestEntryKeeper(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
limit int64
|
||||
run func(t *testing.T, k *entryKeeper, in *countingObjects)
|
||||
wantGets int
|
||||
wantDrops int
|
||||
}{
|
||||
{"second Get of an entry is from memory", 1000, func(t *testing.T, k *entryKeeper, in *countingObjects) {
|
||||
in.data["entry.a"] = []byte("A")
|
||||
for i := 0; i < 3; i++ {
|
||||
if d, err := k.Get(ctx, "entry.a"); err != nil || string(d) != "A" {
|
||||
t.Fatalf("get: %q %v", d, err)
|
||||
}
|
||||
}
|
||||
}, 1, 0},
|
||||
{"a Put entry is kept, never read", 1000, func(t *testing.T, k *entryKeeper, in *countingObjects) {
|
||||
if err := k.Put(ctx, "entry.a", []byte("A")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if d, _ := k.Get(ctx, "entry.a"); string(d) != "A" {
|
||||
t.Fatalf("got %q", d)
|
||||
}
|
||||
}, 0, 0},
|
||||
{"Delete drops it and the next Get is an error naming no object", 1000, func(t *testing.T, k *entryKeeper, in *countingObjects) {
|
||||
_ = k.Put(ctx, "entry.a", []byte("A"))
|
||||
_ = k.Delete(ctx, "entry.a")
|
||||
if _, err := k.Get(ctx, "entry.a"); !errors.Is(err, ErrNoObject) {
|
||||
t.Fatalf("err %v", err)
|
||||
}
|
||||
}, 1, 0},
|
||||
{"a name that is no entry always reaches the bus", 1000, func(t *testing.T, k *entryKeeper, in *countingObjects) {
|
||||
in.data["trunk.1"] = []byte("L")
|
||||
_, _ = k.Get(ctx, "trunk.1")
|
||||
_, _ = k.Get(ctx, "trunk.1")
|
||||
}, 2, 0},
|
||||
{"over the bound everything kept is dropped and said", 3, func(t *testing.T, k *entryKeeper, in *countingObjects) {
|
||||
in.data["entry.a"], in.data["entry.b"] = []byte("AA"), []byte("BB")
|
||||
_, _ = k.Get(ctx, "entry.a")
|
||||
_, _ = k.Get(ctx, "entry.b") // drops a
|
||||
_, _ = k.Get(ctx, "entry.a") // from the bus again
|
||||
}, 3, 2},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
in := &countingObjects{data: map[string][]byte{}}
|
||||
drops := 0
|
||||
k := newEntryKeeper(in, tc.limit, func(string, ...any) { drops++ })
|
||||
tc.run(t, k, in)
|
||||
if in.gets != tc.wantGets || drops != tc.wantDrops {
|
||||
t.Fatalf("bus gets %d (want %d), drops said %d (want %d)", in.gets, tc.wantGets, drops, tc.wantDrops)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user