Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a12c194f6c |
@@ -285,7 +285,9 @@ func main() {
|
||||
acts := meshhandactsclient.Recorder{Ask: stdio.Ask}
|
||||
tools := Tools(asks, time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState)), acts)
|
||||
// The modules and the module graph are the newest catalog graph revision's (milestone 47/G): revision.go.
|
||||
revisions := bundleObjects{busstate.ObjectsOf(index.CatalogRevisionsObjects)}
|
||||
revisions := index.KeepEntries(bundleObjects{busstate.ObjectsOf(index.CatalogRevisionsObjects)}, index.EntryKeptBytes, func(format string, a ...any) {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] "+format+"\n", a...)
|
||||
})
|
||||
trunk := index.TrunkStates{Objects: revisions, Under: keptSource, Kept: &index.KeptTrunk{Say: func(format string, a ...any) {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] "+format+"\n", a...)
|
||||
}}}
|
||||
|
||||
@@ -356,7 +356,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// The view of the trunk is the newest trunk revision of the catalog graph the objects keep (milestone 47/G, ADR 0656 §6:
|
||||
// one form of the fact), read once here and kept as a revision after every write; objects not on the bus yet, or none
|
||||
// kept, start it empty, and the first start's catch-up takes the trunk in once the gate's source is read (catchUp).
|
||||
revisions := newBusObjects(js, cred, index.CatalogRevisionsObjects)
|
||||
revisions := index.KeepEntries(newBusObjects(js, cred, index.CatalogRevisionsObjects), index.EntryKeptBytes, logger.Printf)
|
||||
modules, graph, at, err := index.TrunkView(ctx, revisions)
|
||||
switch {
|
||||
case errors.Is(err, process.ErrNotYet):
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// EntryKeptBytes bounds the bytes of entry objects the indexer holds in memory.
|
||||
const EntryKeptBytes = 64 << 20
|
||||
|
||||
// entryKeeper is the catalog graph's objects with every entry object (entry.<digest>) kept in memory once read or written:
|
||||
// an entry is written once and never changes, and every revision that lists it shares it, so the indexer reads each entry
|
||||
// from the bus once per process (design 47 G §354, A-252). Revision lists and every other name go to the bus as before. A
|
||||
// Delete drops the entry, so a revision that still lists it fails naming it, as before.
|
||||
type entryKeeper struct {
|
||||
Objects
|
||||
limit int64
|
||||
logf func(string, ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
kept map[string][]byte
|
||||
size int64
|
||||
// Hits and Reads count the entry Gets answered from memory and from the bus.
|
||||
hits, reads int64
|
||||
}
|
||||
|
||||
func newEntryKeeper(inner Objects, limit int64, logf func(string, ...any)) *entryKeeper {
|
||||
return &entryKeeper{Objects: inner, limit: limit, logf: logf, kept: map[string][]byte{}}
|
||||
}
|
||||
|
||||
func isEntry(name string) bool { return strings.HasPrefix(name, "entry.") }
|
||||
|
||||
// keep holds data under name; over the bound it drops everything kept first and says so once per drop.
|
||||
func (k *entryKeeper) keep(name string, data []byte) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
if old, ok := k.kept[name]; ok {
|
||||
k.size -= int64(len(old))
|
||||
delete(k.kept, name)
|
||||
}
|
||||
if int64(len(data)) > k.limit {
|
||||
return
|
||||
}
|
||||
if k.size+int64(len(data)) > k.limit {
|
||||
k.logf("the %d entry objects kept in memory (%d bytes) pass the bound of %d bytes: dropped, they are read from the bus again", len(k.kept), k.size, k.limit)
|
||||
k.kept, k.size = map[string][]byte{}, 0
|
||||
}
|
||||
k.kept[name] = data
|
||||
k.size += int64(len(data))
|
||||
}
|
||||
|
||||
func (k *entryKeeper) drop(name string) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
if old, ok := k.kept[name]; ok {
|
||||
k.size -= int64(len(old))
|
||||
delete(k.kept, name)
|
||||
}
|
||||
}
|
||||
|
||||
func (k *entryKeeper) Get(ctx context.Context, name string) ([]byte, error) {
|
||||
if !isEntry(name) {
|
||||
return k.Objects.Get(ctx, name)
|
||||
}
|
||||
k.mu.Lock()
|
||||
data, ok := k.kept[name]
|
||||
if ok {
|
||||
k.hits++
|
||||
}
|
||||
k.mu.Unlock()
|
||||
if ok {
|
||||
return data, nil
|
||||
}
|
||||
data, err := k.Objects.Get(ctx, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
k.mu.Lock()
|
||||
k.reads++
|
||||
k.mu.Unlock()
|
||||
k.keep(name, data)
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func (k *entryKeeper) Put(ctx context.Context, name string, data []byte) error {
|
||||
if err := k.Objects.Put(ctx, name, data); err != nil {
|
||||
if isEntry(name) {
|
||||
k.drop(name)
|
||||
}
|
||||
return err
|
||||
}
|
||||
if isEntry(name) {
|
||||
k.keep(name, append([]byte(nil), data...))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (k *entryKeeper) Delete(ctx context.Context, name string) error {
|
||||
if isEntry(name) {
|
||||
k.drop(name)
|
||||
}
|
||||
err := k.Objects.Delete(ctx, name)
|
||||
if isEntry(name) {
|
||||
k.drop(name)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// KeepEntries is inner with its entry objects kept in memory (see entryKeeper), up to limit bytes; logf says a drop.
|
||||
func KeepEntries(inner Objects, limit int64, logf func(string, ...any)) Objects {
|
||||
return newEntryKeeper(inner, limit, logf)
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type countingObjects struct {
|
||||
Objects
|
||||
data map[string][]byte
|
||||
gets int
|
||||
}
|
||||
|
||||
func (c *countingObjects) Put(_ context.Context, n string, d []byte) error { c.data[n] = d; return nil }
|
||||
func (c *countingObjects) Get(_ context.Context, n string) ([]byte, error) {
|
||||
c.gets++
|
||||
d, ok := c.data[n]
|
||||
if !ok {
|
||||
return nil, ErrNoObject
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
func (c *countingObjects) Delete(_ context.Context, n string) error { delete(c.data, n); return nil }
|
||||
|
||||
func TestEntryKeeper(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
limit int64
|
||||
run func(t *testing.T, k *entryKeeper, in *countingObjects)
|
||||
wantGets int
|
||||
wantDrops int
|
||||
}{
|
||||
{"second Get of an entry is from memory", 1000, func(t *testing.T, k *entryKeeper, in *countingObjects) {
|
||||
in.data["entry.a"] = []byte("A")
|
||||
for i := 0; i < 3; i++ {
|
||||
if d, err := k.Get(ctx, "entry.a"); err != nil || string(d) != "A" {
|
||||
t.Fatalf("get: %q %v", d, err)
|
||||
}
|
||||
}
|
||||
}, 1, 0},
|
||||
{"a Put entry is kept, never read", 1000, func(t *testing.T, k *entryKeeper, in *countingObjects) {
|
||||
if err := k.Put(ctx, "entry.a", []byte("A")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if d, _ := k.Get(ctx, "entry.a"); string(d) != "A" {
|
||||
t.Fatalf("got %q", d)
|
||||
}
|
||||
}, 0, 0},
|
||||
{"Delete drops it and the next Get is an error naming no object", 1000, func(t *testing.T, k *entryKeeper, in *countingObjects) {
|
||||
_ = k.Put(ctx, "entry.a", []byte("A"))
|
||||
_ = k.Delete(ctx, "entry.a")
|
||||
if _, err := k.Get(ctx, "entry.a"); !errors.Is(err, ErrNoObject) {
|
||||
t.Fatalf("err %v", err)
|
||||
}
|
||||
}, 1, 0},
|
||||
{"a name that is no entry always reaches the bus", 1000, func(t *testing.T, k *entryKeeper, in *countingObjects) {
|
||||
in.data["trunk.1"] = []byte("L")
|
||||
_, _ = k.Get(ctx, "trunk.1")
|
||||
_, _ = k.Get(ctx, "trunk.1")
|
||||
}, 2, 0},
|
||||
{"over the bound everything kept is dropped and said", 3, func(t *testing.T, k *entryKeeper, in *countingObjects) {
|
||||
in.data["entry.a"], in.data["entry.b"] = []byte("AA"), []byte("BB")
|
||||
_, _ = k.Get(ctx, "entry.a")
|
||||
_, _ = k.Get(ctx, "entry.b") // drops a
|
||||
_, _ = k.Get(ctx, "entry.a") // from the bus again
|
||||
}, 3, 2},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
in := &countingObjects{data: map[string][]byte{}}
|
||||
drops := 0
|
||||
k := newEntryKeeper(in, tc.limit, func(string, ...any) { drops++ })
|
||||
tc.run(t, k, in)
|
||||
if in.gets != tc.wantGets || drops != tc.wantDrops {
|
||||
t.Fatalf("bus gets %d (want %d), drops said %d (want %d)", in.gets, tc.wantGets, drops, tc.wantDrops)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user