Compare commits

...
Author SHA1 Message Date
jschoubben a12c194f6c indexer: the indexer reads each catalog graph entry from the bus once per process
An entry object entry.<digest> is written once and never changes, yet every
revision read (the verb revision on each page, the composition steps, about 20
callers of ReadRevision) took one object Get per entry, each bounded by the bus
client's deadline: composition steps ended "could not run: the step's catalog
revision is not read ... context deadline exceeded". The indexer now keeps the
bytes of entry objects it has read or written in memory (bound 64 MiB, dropped
whole and logged once when passed); a Delete drops the entry, so a revision that
still lists it fails naming it. Revision lists are read from the bus as before.

Builds: design 47 G §354 (composition reads the revision), A-252
2026-11-16 22:00:11 +01:00
mesh-tools 38f1c2f10a Merge pull request 'assigner: a composition step reads its catalog revision once and again on a timeout' (#232) from fix/composition-reads-revision-once into main 2026-11-16 20:32:07 +00:00
jschoubben bece2421f1 assigner: a composition step reads its catalog revision once and again on a timeout
Builds: design 47 G §354, A-252, A-261
2026-11-16 21:31:34 +01:00
8 changed files with 492 additions and 30 deletions
@@ -0,0 +1,181 @@
package main
import (
"context"
"encoding/json"
"fmt"
"strings"
"sync"
"time"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-sdk/go/buscalls"
)
// The reads of a composition step (design 47 G §354, A-252, A-261): the entries of one catalog revision are read once and shared by
// every composition step of that revision, since an entry is content-addressed and a revision never changes; a read that times out
// or fails in transport is made again with a growing wait for a bounded time, said in the journal, and only then ends the step as
// could not run.
const (
// readsBound is how long a read is made again before the step could not run.
readsBound = 3 * time.Minute
// readsFirstWait and readsMaxWait bound the wait between two tries.
readsFirstWait = time.Second
readsMaxWait = 15 * time.Second
// readsKept is how many catalog revisions' halves are kept; the oldest read goes first.
readsKept = 8
)
// readKey names one revision of one watched branch.
type readKey struct {
branch string
revision int
}
// readSlot is one revision's read: done closes when half or err is set.
type readSlot struct {
done chan struct{}
half mcc.Half
err error
}
// compositionReads shares the reads of composition steps.
type compositionReads struct {
mu sync.Mutex
revs map[readKey]*readSlot
order []readKey
seats map[string]json.RawMessage // by seat and digest: the definition, raw
}
// transient says whether a read's fault is of the transport or a timeout, so that it is worth reading again.
func transient(err error, said string) bool {
text := said
if err != nil {
text = err.Error() + " " + said
}
text = strings.ToLower(text)
for _, w := range []string{"deadline exceeded", "timeout", "timed out", "no responders", "connection", "i/o", "eof", "broken pipe", "unavailable"} {
if strings.Contains(text, w) {
return true
}
}
return false
}
// callRetry calls the seat verb key, again with a growing wait while its fault is transient, for readsBound; it answers the body of
// a good answer, or the last fault.
func (r *compositionRun) callRetry(ctx context.Context, key string, body any) (json.RawMessage, string, error) {
start := r.now()
wait := readsFirstWait
for try := 1; ; try++ {
raw, err := r.call(key, body)
var inner json.RawMessage
var said string
if err == nil {
var ok bool
if inner, said, ok = buscalls.Unwrap(raw); ok {
return inner, "", nil
}
}
if !transient(err, said) {
return nil, said, err
}
if r.now().Sub(start)+wait > readsBound {
return nil, said, err
}
r.logf("composition step: %s is not read (try %d, %v %s), read again in %s", key, try, err, said, wait)
if serr := r.waitFor(ctx, wait); serr != nil {
return nil, said, serr
}
if wait *= 2; wait > readsMaxWait {
wait = readsMaxWait
}
}
}
// waitFor waits d, or until ctx ends.
func (r *compositionRun) waitFor(ctx context.Context, d time.Duration) error {
if r.sleep != nil {
return r.sleep(ctx, d)
}
t := time.NewTimer(d)
defer t.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-t.C:
return nil
}
}
// half is the revision's half, read once for every step of it: a step that finds it being read waits for that read, and a read that
// failed is not kept, so the next step reads again.
func (r *compositionRun) half(ctx context.Context, key readKey, read func() (mcc.Half, error)) (mcc.Half, error) {
c := r.reads
c.mu.Lock()
if c.revs == nil {
c.revs = map[readKey]*readSlot{}
}
if s, ok := c.revs[key]; ok {
c.mu.Unlock()
select {
case <-s.done:
if s.err == nil {
return s.half, nil
}
case <-ctx.Done():
return mcc.Half{}, ctx.Err()
}
// The read this step waited for failed: read it here (the failed slot is already dropped).
return r.half(ctx, key, read)
}
s := &readSlot{done: make(chan struct{})}
c.revs[key] = s
c.order = append(c.order, key)
for len(c.order) > readsKept {
delete(c.revs, c.order[0])
c.order = c.order[1:]
}
c.mu.Unlock()
s.half, s.err = read()
if s.err != nil {
c.mu.Lock()
if c.revs[key] == s {
delete(c.revs, key)
}
for i, k := range c.order {
if k == key {
c.order = append(c.order[:i], c.order[i+1:]...)
break
}
}
c.mu.Unlock()
}
close(s.done)
return s.half, s.err
}
// seatKept is the definition kept of a seat at a digest.
func (c *compositionReads) seatKept(seat, digest string) (json.RawMessage, bool) {
c.mu.Lock()
defer c.mu.Unlock()
d, ok := c.seats[seat+"@"+digest]
return d, ok
}
func (c *compositionReads) keepSeat(seat, digest string, def json.RawMessage) {
c.mu.Lock()
defer c.mu.Unlock()
if c.seats == nil {
c.seats = map[string]json.RawMessage{}
}
c.seats[seat+"@"+digest] = def
}
func unreadErr(what string, err error, said string) error {
if err != nil {
return fmt.Errorf("%s: %w", what, err)
}
return fmt.Errorf("%s: %s", what, said)
}
@@ -18,7 +18,6 @@ import (
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"git.novox.be/novox/mesh-sdk/go/keys"
)
@@ -122,6 +121,9 @@ type compositionRun struct {
// put puts a run ask on the build seat's check queue under the assigner's name.
put func(ctx context.Context, a buildstep.StepAsk) error
ends *runEnds
// reads shares the catalog reads among steps (compositionreads.go); sleep waits between two tries of a read (nil: a timer).
reads *compositionReads
sleep func(ctx context.Context, d time.Duration) error
// ask asks a seat verb on one node, for the run's answer kept as the build seat's object (buildstep.StepEnded.ReadOutput).
ask buildstep.AskVerb
now func() time.Time
@@ -136,11 +138,11 @@ type machineEnd struct {
// run runs one composition step and answers what it came to: failed when a machine does not compose, could not run when a machine
// could not be composed, else passed with a warning per machine away or not judged.
func (r *compositionRun) run(ctx context.Context, _ string, run meshwants.StepRun) stepOutcome {
half, err := r.revision(run)
half, err := r.revision(ctx, run)
if err != nil {
return couldNotRun("the step's catalog revision is not read: " + err.Error())
}
seats, err := r.seats(half.Revision.Definitions)
seats, err := r.seats(ctx, half.Revision.Definitions)
if err != nil {
return couldNotRun(err.Error())
}
@@ -148,7 +150,7 @@ func (r *compositionRun) run(ctx context.Context, _ string, run meshwants.StepRu
if err != nil {
return couldNotRun(err.Error())
}
programOf, err := r.programs(half)
programOf, err := r.programs(ctx, half)
if err != nil {
return couldNotRun(err.Error())
}
@@ -275,20 +277,24 @@ func verdictOf(node string, answer []byte) machineEnd {
return machineEnd{couldNot: node + ": the composer's verdict has no case " + changeCase}
}
// revision reads the step's catalog revision whole, page by page, through mesh-catalog's `revision` (A-252).
func (r *compositionRun) revision(run meshwants.StepRun) (mcc.Half, error) {
// revision reads the step's catalog revision whole, page by page, through mesh-catalog's `revision` (A-252), once for every step of
// it (compositionreads.go).
func (r *compositionRun) revision(ctx context.Context, run meshwants.StepRun) (mcc.Half, error) {
if run.Revision <= 0 {
return mcc.Half{}, errors.New("the step names no catalog revision")
}
return r.half(ctx, readKey{run.Branch, int(run.Revision)}, func() (mcc.Half, error) { return r.readRevision(ctx, run) })
}
func (r *compositionRun) readRevision(ctx context.Context, run meshwants.StepRun) (mcc.Half, error) {
half := mcc.Half{Entries: map[string]meshcatalog.CatalogEntry{}}
in := meshcatalog.RevisionInput{Branch: keys.EscapeName(run.Branch), Revision: run.Revision}
for {
raw, err := r.call(mcc.RevisionVerb, in)
body, said, err := r.callRetry(ctx, mcc.RevisionVerb, in)
if err != nil {
return mcc.Half{}, err
}
body, said, ok := buscalls.Unwrap(raw)
if !ok {
if body == nil {
return mcc.Half{}, fmt.Errorf("%s answered: %s", mcc.RevisionVerb, said)
}
var a meshcatalog.RevisionAnswer
@@ -314,20 +320,21 @@ func (r *compositionRun) revision(run meshwants.StepRun) (mcc.Half, error) {
// seats are the definitions of the revision's (seat, digest) pairs, raw (A-261): the revision's definitions carry no definition
// yet (A-234), so each is mesh-catalog's seat-definitions answer for the seat where the trunk's digest is the revision's; a digest
// neither holds is an error naming the seat and the digest.
func (r *compositionRun) seats(defs []meshcatalog.CatalogRevisionDefinitionsItem) ([]json.RawMessage, error) {
func (r *compositionRun) seats(ctx context.Context, defs []meshcatalog.CatalogRevisionDefinitionsItem) ([]json.RawMessage, error) {
out := make([]json.RawMessage, 0, len(defs))
for _, d := range defs {
raw, err := r.call(seatDefinitionsKey, meshcatalog.SeatDefinitionsInput{Seat: d.Seat})
if err != nil {
return nil, fmt.Errorf("the definition of the seat %s is not read: %w", d.Seat, err)
if def, ok := r.reads.seatKept(d.Seat, d.Digest); ok { // a definition at a digest never changes
out = append(out, def)
continue
}
body, said, ok := buscalls.Unwrap(raw)
if !ok {
return nil, fmt.Errorf("the definition of the seat %s is not read: %s", d.Seat, said)
what := "the definition of the seat " + d.Seat + " is not read"
body, said, err := r.callRetry(ctx, seatDefinitionsKey, meshcatalog.SeatDefinitionsInput{Seat: d.Seat})
if err != nil || body == nil {
return nil, unreadErr(what, err, said)
}
var a meshcatalog.SeatDefinitionsAnswer
if err := json.Unmarshal(body, &a); err != nil {
return nil, fmt.Errorf("the definition of the seat %s is not read: %w", d.Seat, err)
return nil, fmt.Errorf("%s: %w", what, err)
}
if a.Entry == nil || a.Entry.Digest != d.Digest || a.Entry.Definition == nil {
return nil, fmt.Errorf("no definition of the seat %s with the digest %s is held: neither the revision nor the trunk holds it", d.Seat, d.Digest)
@@ -336,6 +343,7 @@ func (r *compositionRun) seats(defs []meshcatalog.CatalogRevisionDefinitionsItem
if err != nil {
return nil, err
}
r.reads.keepSeat(d.Seat, d.Digest, def)
out = append(out, def)
}
return out, nil
@@ -412,7 +420,7 @@ func programOfEntry(e meshcatalog.CatalogEntry) (buildstep.RunProgram, error) {
// programs answers, per machine, the composer its run builds (A-226, A-243, A-252): where the revision's node-engine entry differs
// from the trunk's, that build for every machine; else the build the machine's node-engine runs, as its `running` says, its entry
// read through mesh-catalog's `entry`; a machine whose node-engine's `running` is not heard gets the revision's entry, said (A-451).
func (r *compositionRun) programs(half mcc.Half) (func(node string) (buildstep.RunProgram, error), error) {
func (r *compositionRun) programs(ctx context.Context, half mcc.Half) (func(node string) (buildstep.RunProgram, error), error) {
engine, ok := nodeEngineEntry(half.Entries)
if !ok {
return nil, fmt.Errorf("the catalog revision %d holds no module claiming the seat %s", half.Revision.Revision, ne.Seat)
@@ -436,17 +444,14 @@ func (r *compositionRun) programs(half mcc.Half) (func(node string) (buildstep.R
if ok {
return p, nil
}
raw, err := r.call(entryVerbKey, meshcatalog.EntryInput{Fingerprint: running.Fingerprint})
if err != nil {
return buildstep.RunProgram{}, fmt.Errorf("the entry of the node-engine's build %s is not read: %w", running.Fingerprint, err)
}
body, said, ok := buscalls.Unwrap(raw)
if !ok {
return buildstep.RunProgram{}, fmt.Errorf("the entry of the node-engine's build %s is not read: %s", running.Fingerprint, said)
what := "the entry of the node-engine's build " + running.Fingerprint + " is not read"
body, said, err := r.callRetry(ctx, entryVerbKey, meshcatalog.EntryInput{Fingerprint: running.Fingerprint})
if err != nil || body == nil {
return buildstep.RunProgram{}, unreadErr(what, err, said)
}
var a meshcatalog.EntryAnswer
if err := json.Unmarshal(body, &a); err != nil {
return buildstep.RunProgram{}, fmt.Errorf("the entry of the node-engine's build %s is not read: %w", running.Fingerprint, err)
return buildstep.RunProgram{}, fmt.Errorf("%s: %w", what, err)
}
p, err = programOfEntry(a.Entry)
if err != nil {
@@ -82,7 +82,7 @@ func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) *
ends := &runEnds{}
s.r = &compositionRun{
inputs: &compositionInputs{machines: func(string, string) []string { return []string{"ace", "g14", "shanks"} }, callFile: engines, bound: time.Second},
keeper: keeper, call: call, ends: ends, away: away, now: time.Now, logf: t.Logf,
keeper: keeper, reads: &compositionReads{}, call: call, ends: ends, away: away, now: time.Now, logf: t.Logf,
trunk: func() (mcc.Half, bool) {
return mcc.Half{Entries: map[string]meshcatalog.CatalogEntry{"engine": engine(trunkEngine, commitE1)}}, true
},
@@ -187,3 +187,84 @@ func TestTheCompositionStepRunsOneAskPerMachine(t *testing.T) {
t.Errorf("an ask at the cap came to %+v, %d asks", out, len(full.asks))
}
}
// Design 47 G §354, A-252: the entries of one catalog revision are read once for every step of it, and a read that times out is
// made again with a growing wait for a bounded time, said in the journal, before the step could not run.
func TestTheCompositionStepReadsTheRevisionOnceAndAgainOnATimeout(t *testing.T) {
step := meshwants.StepRun{Module: "m", Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition}
for _, tc := range []struct {
name string
faults int // consecutive faults of the `revision` read before it answers
fault error // the fault
steps int
want string // the outcome of the last step
revisionCalls int
waits int
}{
{"two steps of one revision read it once", 0, nil, 2, meshwants.StepEndedOutcomeFailed, 2, 0},
{"a timeout is read again", 3, errors.New("context deadline exceeded"), 1, meshwants.StepEndedOutcomeFailed, 5, 3},
{"a fault that stays ends could not run", 1000, errors.New("nats: timeout"), 1, meshwants.StepEndedOutcomeCouldNotRun, -1, -1},
{"a fault of the answer is not read again", 1, errors.New("no such revision"), 1, meshwants.StepEndedOutcomeCouldNotRun, 1, 0},
} {
t.Run(tc.name, func(t *testing.T) {
s := newRunStand(t, "src2:e1", "src2:e1", "d1")
inner := s.r.call
var mu sync.Mutex
calls, faults := 0, tc.faults
s.r.call = func(key string, body any) (json.RawMessage, error) {
if key == mcc.RevisionVerb {
mu.Lock()
calls++
f := faults > 0
faults--
mu.Unlock()
if f {
return nil, tc.fault
}
}
return inner(key, body)
}
clock := time.Unix(1000, 0)
var waits []time.Duration
s.r.now = func() time.Time { mu.Lock(); defer mu.Unlock(); return clock }
s.r.sleep = func(_ context.Context, d time.Duration) error {
mu.Lock()
defer mu.Unlock()
waits = append(waits, d)
clock = clock.Add(d)
return nil
}
var wg sync.WaitGroup
outs := make([]stepOutcome, tc.steps)
for i := range outs {
wg.Add(1)
go func() { defer wg.Done(); outs[i] = s.r.run(context.Background(), "ask-step", step) }()
}
wg.Wait()
for _, o := range outs {
if o.Outcome != tc.want {
t.Fatalf("a step came to %+v, want %s", o, tc.want)
}
}
if tc.waits == -1 { // never answers: bounded, growing waits
if len(waits) < 3 || waits[0] != readsFirstWait || waits[1] != 2*readsFirstWait || waits[len(waits)-1] != readsMaxWait {
t.Errorf("the waits %v do not grow to %s", waits, readsMaxWait)
}
var total time.Duration
for _, w := range waits {
total += w
}
if total > readsBound {
t.Errorf("read again for %s, bound %s", total, readsBound)
}
if !strings.Contains(outs[0].Said, "catalog revision is not read") || !strings.Contains(outs[0].Said, "timeout") {
t.Errorf("the step said %q", outs[0].Said)
}
return
}
if calls != tc.revisionCalls || len(waits) != tc.waits {
t.Errorf("`revision` asked %d times with %d waits, want %d and %d", calls, len(waits), tc.revisionCalls, tc.waits)
}
})
}
}
@@ -675,7 +675,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
}
// The composition step (design 47 G §22, §354): the seat mesh-wants' accepted kind step, run by one run ask per machine.
if bus != nil && js != nil && inputs != nil && p.publishWants != nil {
run := &compositionRun{inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFile}, keeper: inputs, call: bus.CallKept,
run := &compositionRun{inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFile}, keeper: inputs, reads: &compositionReads{}, call: bus.CallKept,
trunk: trunkOf(iv, w.trunkName), running: w.runningOf, away: away, put: runAskPublisher(js), ends: ends, ask: bus.CallFile, now: time.Now, logf: logger.Printf}
steps := &compositionSteps{node: thisNode, say: p.publishWants, run: run.run, now: time.Now, logf: logger.Printf,
asker: stepAskerOf(seatlist.Definition(gate.Reader, meshwants.Seat))}
@@ -285,7 +285,9 @@ func main() {
acts := meshhandactsclient.Recorder{Ask: stdio.Ask}
tools := Tools(asks, time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState)), acts)
// The modules and the module graph are the newest catalog graph revision's (milestone 47/G): revision.go.
revisions := bundleObjects{busstate.ObjectsOf(index.CatalogRevisionsObjects)}
revisions := index.KeepEntries(bundleObjects{busstate.ObjectsOf(index.CatalogRevisionsObjects)}, index.EntryKeptBytes, func(format string, a ...any) {
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] "+format+"\n", a...)
})
trunk := index.TrunkStates{Objects: revisions, Under: keptSource, Kept: &index.KeptTrunk{Say: func(format string, a ...any) {
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] "+format+"\n", a...)
}}}
@@ -356,7 +356,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// The view of the trunk is the newest trunk revision of the catalog graph the objects keep (milestone 47/G, ADR 0656 §6:
// one form of the fact), read once here and kept as a revision after every write; objects not on the bus yet, or none
// kept, start it empty, and the first start's catch-up takes the trunk in once the gate's source is read (catchUp).
revisions := newBusObjects(js, cred, index.CatalogRevisionsObjects)
revisions := index.KeepEntries(newBusObjects(js, cred, index.CatalogRevisionsObjects), index.EntryKeptBytes, logger.Printf)
modules, graph, at, err := index.TrunkView(ctx, revisions)
switch {
case errors.Is(err, process.ErrNotYet):
@@ -0,0 +1,113 @@
package index
import (
"context"
"strings"
"sync"
)
// EntryKeptBytes bounds the bytes of entry objects the indexer holds in memory.
const EntryKeptBytes = 64 << 20
// entryKeeper is the catalog graph's objects with every entry object (entry.<digest>) kept in memory once read or written:
// an entry is written once and never changes, and every revision that lists it shares it, so the indexer reads each entry
// from the bus once per process (design 47 G §354, A-252). Revision lists and every other name go to the bus as before. A
// Delete drops the entry, so a revision that still lists it fails naming it, as before.
type entryKeeper struct {
Objects
limit int64
logf func(string, ...any)
mu sync.Mutex
kept map[string][]byte
size int64
// Hits and Reads count the entry Gets answered from memory and from the bus.
hits, reads int64
}
func newEntryKeeper(inner Objects, limit int64, logf func(string, ...any)) *entryKeeper {
return &entryKeeper{Objects: inner, limit: limit, logf: logf, kept: map[string][]byte{}}
}
func isEntry(name string) bool { return strings.HasPrefix(name, "entry.") }
// keep holds data under name; over the bound it drops everything kept first and says so once per drop.
func (k *entryKeeper) keep(name string, data []byte) {
k.mu.Lock()
defer k.mu.Unlock()
if old, ok := k.kept[name]; ok {
k.size -= int64(len(old))
delete(k.kept, name)
}
if int64(len(data)) > k.limit {
return
}
if k.size+int64(len(data)) > k.limit {
k.logf("the %d entry objects kept in memory (%d bytes) pass the bound of %d bytes: dropped, they are read from the bus again", len(k.kept), k.size, k.limit)
k.kept, k.size = map[string][]byte{}, 0
}
k.kept[name] = data
k.size += int64(len(data))
}
func (k *entryKeeper) drop(name string) {
k.mu.Lock()
defer k.mu.Unlock()
if old, ok := k.kept[name]; ok {
k.size -= int64(len(old))
delete(k.kept, name)
}
}
func (k *entryKeeper) Get(ctx context.Context, name string) ([]byte, error) {
if !isEntry(name) {
return k.Objects.Get(ctx, name)
}
k.mu.Lock()
data, ok := k.kept[name]
if ok {
k.hits++
}
k.mu.Unlock()
if ok {
return data, nil
}
data, err := k.Objects.Get(ctx, name)
if err != nil {
return nil, err
}
k.mu.Lock()
k.reads++
k.mu.Unlock()
k.keep(name, data)
return data, nil
}
func (k *entryKeeper) Put(ctx context.Context, name string, data []byte) error {
if err := k.Objects.Put(ctx, name, data); err != nil {
if isEntry(name) {
k.drop(name)
}
return err
}
if isEntry(name) {
k.keep(name, append([]byte(nil), data...))
}
return nil
}
func (k *entryKeeper) Delete(ctx context.Context, name string) error {
if isEntry(name) {
k.drop(name)
}
err := k.Objects.Delete(ctx, name)
if isEntry(name) {
k.drop(name)
}
return err
}
// KeepEntries is inner with its entry objects kept in memory (see entryKeeper), up to limit bytes; logf says a drop.
func KeepEntries(inner Objects, limit int64, logf func(string, ...any)) Objects {
return newEntryKeeper(inner, limit, logf)
}
@@ -0,0 +1,80 @@
package index
import (
"context"
"errors"
"testing"
)
type countingObjects struct {
Objects
data map[string][]byte
gets int
}
func (c *countingObjects) Put(_ context.Context, n string, d []byte) error { c.data[n] = d; return nil }
func (c *countingObjects) Get(_ context.Context, n string) ([]byte, error) {
c.gets++
d, ok := c.data[n]
if !ok {
return nil, ErrNoObject
}
return d, nil
}
func (c *countingObjects) Delete(_ context.Context, n string) error { delete(c.data, n); return nil }
func TestEntryKeeper(t *testing.T) {
ctx := context.Background()
for _, tc := range []struct {
name string
limit int64
run func(t *testing.T, k *entryKeeper, in *countingObjects)
wantGets int
wantDrops int
}{
{"second Get of an entry is from memory", 1000, func(t *testing.T, k *entryKeeper, in *countingObjects) {
in.data["entry.a"] = []byte("A")
for i := 0; i < 3; i++ {
if d, err := k.Get(ctx, "entry.a"); err != nil || string(d) != "A" {
t.Fatalf("get: %q %v", d, err)
}
}
}, 1, 0},
{"a Put entry is kept, never read", 1000, func(t *testing.T, k *entryKeeper, in *countingObjects) {
if err := k.Put(ctx, "entry.a", []byte("A")); err != nil {
t.Fatal(err)
}
if d, _ := k.Get(ctx, "entry.a"); string(d) != "A" {
t.Fatalf("got %q", d)
}
}, 0, 0},
{"Delete drops it and the next Get is an error naming no object", 1000, func(t *testing.T, k *entryKeeper, in *countingObjects) {
_ = k.Put(ctx, "entry.a", []byte("A"))
_ = k.Delete(ctx, "entry.a")
if _, err := k.Get(ctx, "entry.a"); !errors.Is(err, ErrNoObject) {
t.Fatalf("err %v", err)
}
}, 1, 0},
{"a name that is no entry always reaches the bus", 1000, func(t *testing.T, k *entryKeeper, in *countingObjects) {
in.data["trunk.1"] = []byte("L")
_, _ = k.Get(ctx, "trunk.1")
_, _ = k.Get(ctx, "trunk.1")
}, 2, 0},
{"over the bound everything kept is dropped and said", 3, func(t *testing.T, k *entryKeeper, in *countingObjects) {
in.data["entry.a"], in.data["entry.b"] = []byte("AA"), []byte("BB")
_, _ = k.Get(ctx, "entry.a")
_, _ = k.Get(ctx, "entry.b") // drops a
_, _ = k.Get(ctx, "entry.a") // from the bus again
}, 3, 2},
} {
t.Run(tc.name, func(t *testing.T) {
in := &countingObjects{data: map[string][]byte{}}
drops := 0
k := newEntryKeeper(in, tc.limit, func(string, ...any) { drops++ })
tc.run(t, k, in)
if in.gets != tc.wantGets || drops != tc.wantDrops {
t.Fatalf("bus gets %d (want %d), drops said %d (want %d)", in.gets, tc.wantGets, drops, tc.wantDrops)
}
})
}
}