Merge pull request 'Indexer: an input leaves nothing in its object store; a full store is said by name and fill' (#220) from fix/indexer-inputs-retention into main
This commit was merged in pull request #220.
This commit is contained in:
@@ -6,6 +6,7 @@ import (
|
||||
"crypto/rand"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -13,6 +14,7 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
"git.novox.be/novox/mesh-sdk/go/stateobjects"
|
||||
)
|
||||
@@ -20,12 +22,22 @@ import (
|
||||
type objectsMem struct {
|
||||
mu sync.Mutex
|
||||
objs map[string][]byte
|
||||
cap int // the most bytes the bucket holds; zero for no limit (a full bucket refuses: maximum bytes exceeded)
|
||||
}
|
||||
|
||||
func (*objectsMem) MostBytes() int { return 0 }
|
||||
func (o *objectsMem) Put(name string, data []byte) error {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
if o.cap > 0 {
|
||||
used := len(data)
|
||||
for _, d := range o.objs {
|
||||
used += len(d)
|
||||
}
|
||||
if used > o.cap {
|
||||
return errors.New("nats: API error: code=503 err_code=10077 description=maximum bytes exceeded")
|
||||
}
|
||||
}
|
||||
o.objs[name] = data
|
||||
return nil
|
||||
}
|
||||
@@ -38,8 +50,21 @@ func (o *objectsMem) Get(name string) ([]byte, error) {
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
func (o *objectsMem) Delete(name string) error { return nil }
|
||||
func (o *objectsMem) List() ([]stateobjects.Object, error) { return nil, nil }
|
||||
func (o *objectsMem) Delete(name string) error {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
delete(o.objs, name)
|
||||
return nil
|
||||
}
|
||||
func (o *objectsMem) List() ([]stateobjects.Object, error) {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
var out []stateobjects.Object
|
||||
for n, d := range o.objs {
|
||||
out = append(out, stateobjects.Object{Name: n, Size: int64(len(d))})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// lineMem is a core bus in memory: a chunk published is kept, and its acknowledgement heard at once.
|
||||
type lineMem struct {
|
||||
@@ -131,4 +156,19 @@ func TestAnInputOverOneMiBIsKeptAsAnObjectAndAnsweredInChunks(t *testing.T) {
|
||||
if _, err := verb("", json.RawMessage(`{"digest":"`+digest+`","call":"x"}`)); err == nil {
|
||||
t.Error("a call of no token is answered")
|
||||
}
|
||||
|
||||
// the bucket holds only inputs still asked: the ended ask's object goes with its key, and one a past delete left goes at the next write
|
||||
if err := ix.HearInputEnded(ctx, nodebuildgo.StepEnded{Ask: "ask-big"}); err != nil || len(objs.objs) != 0 {
|
||||
t.Fatalf("an ended ask's input leaves its object: %d left, %v", len(objs.objs), err)
|
||||
}
|
||||
objs.objs["inputs.orphan"] = make([]byte, 100)
|
||||
objs.cap = len(body)*4/3 + 400 // the base64 text, and the orphan below it
|
||||
if _, err := ix.KeepInput(ctx, "ask-two", body); err != nil || len(objs.objs) != 1 {
|
||||
t.Fatalf("a write first removes the objects no key names: %d objects, %v", len(objs.objs), err)
|
||||
}
|
||||
// a full bucket is said loudly, naming the bucket and its fill
|
||||
_, err = ix.KeepInput(ctx, "ask-three", append([]byte{1}, body...))
|
||||
if err == nil || !strings.Contains(err.Error(), "inputs-objects") || !strings.Contains(err.Error(), "1 objects") {
|
||||
t.Fatalf("a refused write names the bucket and its fill: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -65,9 +65,18 @@ func (ix *Indexer) KeepInput(ctx context.Context, ask string, body []byte) (stri
|
||||
if err := ix.sweepInputs(ctx); err != nil {
|
||||
ix.logf("inputs: the keys older than a day are not all deleted: %v", err)
|
||||
}
|
||||
// the object store holds only inputs still asked: an object no key names any longer (one a delete of its key left) goes first
|
||||
if ix.InputObjects != nil {
|
||||
if _, err := inputValues(ix.Inputs, ix.InputObjects).DropKeyGone(); err != nil {
|
||||
ix.logf("inputs: the objects no key names are not all removed: %v", err)
|
||||
}
|
||||
}
|
||||
digest := runDigest(body)
|
||||
key := meshcatalog.InputsByTimeDigestKey(ix.now().UTC().Format(inputsTime), digest)
|
||||
if err := inputValues(ix.Inputs, ix.InputObjects).Put(key, base64.StdEncoding.EncodeToString(body)); err != nil && !errors.Is(err, stateobjects.ErrChanged) {
|
||||
if ix.InputObjects != nil && strings.HasPrefix(err.Error(), "the object for ") {
|
||||
return "", fmt.Errorf("the object store %s refused the input of %d bytes, holding %s: %w", catalogue.BusName(ix.Module, catalogue.StateObjectsName(InputsState)), len(body), ix.inputsFill(), err)
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
if err := ix.Inputs.Create(ctx, InputsAskPrefix+keys.EscapeName(ask), []byte(key)); err != nil && !errors.Is(err, ErrConflict) {
|
||||
@@ -76,6 +85,34 @@ func (ix *Indexer) KeepInput(ctx context.Context, ask string, body []byte) (stri
|
||||
return digest, nil
|
||||
}
|
||||
|
||||
// inputsFill says what the object store of the inputs holds: its objects and their bytes.
|
||||
func (ix *Indexer) inputsFill() string {
|
||||
objs, err := ix.InputObjects.List()
|
||||
if err != nil {
|
||||
return fmt.Sprintf("an unreadable fill (%v)", err)
|
||||
}
|
||||
var total int64
|
||||
for _, o := range objs {
|
||||
total += o.Size
|
||||
}
|
||||
return fmt.Sprintf("%d objects of %d bytes", len(objs), total)
|
||||
}
|
||||
|
||||
// deleteInput deletes the input's key and the object its value names, if any: an input leaves nothing in the object store.
|
||||
func (ix *Indexer) deleteInput(ctx context.Context, key string) error {
|
||||
raw, rev, found, err := ix.Inputs.Get(ctx, key)
|
||||
if err != nil || !found {
|
||||
return err
|
||||
}
|
||||
if err := ix.Inputs.Delete(ctx, key, rev); err != nil {
|
||||
return err
|
||||
}
|
||||
if ref, isRef := stateobjects.ReadRef(raw); isRef && ix.InputObjects != nil {
|
||||
return ix.InputObjects.Delete(ref.Object)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// HearInputEnded deletes the input kept for a run ask when the build seat's step-ended of that ask is heard (§354, A-78 d); an ask
|
||||
// that kept none, or whose input a sweep deleted, changes nothing.
|
||||
func (ix *Indexer) HearInputEnded(ctx context.Context, e nodebuildgo.StepEnded) error {
|
||||
@@ -87,12 +124,8 @@ func (ix *Indexer) HearInputEnded(ctx context.Context, e nodebuildgo.StepEnded)
|
||||
if err != nil || !found {
|
||||
return err
|
||||
}
|
||||
if _, irev, ok, err := ix.Inputs.Get(ctx, string(raw)); err != nil {
|
||||
if err := ix.deleteInput(ctx, string(raw)); err != nil {
|
||||
return err
|
||||
} else if ok {
|
||||
if err := ix.Inputs.Delete(ctx, string(raw), irev); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return ix.Inputs.Delete(ctx, askKey, rev)
|
||||
}
|
||||
@@ -122,10 +155,8 @@ func (ix *Indexer) sweepInputs(ctx context.Context) error {
|
||||
if !ok || perr != nil || ix.now().Sub(at) < InputsKeptFor {
|
||||
continue
|
||||
}
|
||||
if _, rev, found, err := ix.Inputs.Get(ctx, k); err == nil && found {
|
||||
if err := ix.Inputs.Delete(ctx, k, rev); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
if err := ix.deleteInput(ctx, k); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
// a name by ask whose input a sweep or its step-ended deleted goes with it
|
||||
|
||||
Reference in New Issue
Block a user