The indexer's inputs-objects bucket filled twice: deleting an input's key (step-ended, the day sweep) left its object, which only the person's drop tool removed. The input's key and object now go together, each write first drops the objects no key names, and a write the store refuses names the store and its fill. Builds: design 47 G §354, §712, A-237
175 lines
5.6 KiB
Go
175 lines
5.6 KiB
Go
package index
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/rand"
|
|
"encoding/json"
|
|
"errors"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/nats-io/nats.go"
|
|
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
|
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
|
"git.novox.be/novox/mesh-sdk/go/stateobjects"
|
|
)
|
|
|
|
type objectsMem struct {
|
|
mu sync.Mutex
|
|
objs map[string][]byte
|
|
cap int // the most bytes the bucket holds; zero for no limit (a full bucket refuses: maximum bytes exceeded)
|
|
}
|
|
|
|
func (*objectsMem) MostBytes() int { return 0 }
|
|
func (o *objectsMem) Put(name string, data []byte) error {
|
|
o.mu.Lock()
|
|
defer o.mu.Unlock()
|
|
if o.cap > 0 {
|
|
used := len(data)
|
|
for _, d := range o.objs {
|
|
used += len(d)
|
|
}
|
|
if used > o.cap {
|
|
return errors.New("nats: API error: code=503 err_code=10077 description=maximum bytes exceeded")
|
|
}
|
|
}
|
|
o.objs[name] = data
|
|
return nil
|
|
}
|
|
func (o *objectsMem) Get(name string) ([]byte, error) {
|
|
o.mu.Lock()
|
|
defer o.mu.Unlock()
|
|
d, ok := o.objs[name]
|
|
if !ok {
|
|
return nil, errors.New("no such object")
|
|
}
|
|
return d, nil
|
|
}
|
|
func (o *objectsMem) Delete(name string) error {
|
|
o.mu.Lock()
|
|
defer o.mu.Unlock()
|
|
delete(o.objs, name)
|
|
return nil
|
|
}
|
|
func (o *objectsMem) List() ([]stateobjects.Object, error) {
|
|
o.mu.Lock()
|
|
defer o.mu.Unlock()
|
|
var out []stateobjects.Object
|
|
for n, d := range o.objs {
|
|
out = append(out, stateobjects.Object{Name: n, Size: int64(len(d))})
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// lineMem is a core bus in memory: a chunk published is kept, and its acknowledgement heard at once.
|
|
type lineMem struct {
|
|
mu sync.Mutex
|
|
hear nats.MsgHandler
|
|
chunks map[string][]byte
|
|
}
|
|
|
|
func (l *lineMem) Publish(subject string, data []byte) error {
|
|
l.mu.Lock()
|
|
l.chunks[subject] = data
|
|
h := l.hear
|
|
l.mu.Unlock()
|
|
var c buscalls.SeatChunk
|
|
if err := json.Unmarshal(data, &c); err != nil {
|
|
return err
|
|
}
|
|
go h(&nats.Msg{Subject: buscalls.SeatChunks{Seat: meshcatalog.Seat, Verb: meshcatalog.VerbInput, Call: "0123456789abcdef"}.Ack(c.N)})
|
|
return nil
|
|
}
|
|
|
|
func (l *lineMem) Subscribe(_ string, fn nats.MsgHandler) (*nats.Subscription, error) {
|
|
l.mu.Lock()
|
|
l.hear = fn
|
|
l.mu.Unlock()
|
|
return nil, nil
|
|
}
|
|
|
|
// Design 47 G §712, A-237: an input is of any size: over a key's limit it is kept as an object beside the state, and the verb
|
|
// input answers it in chunks of at most 512 KiB, each acknowledged before the next.
|
|
func TestAnInputOverOneMiBIsKeptAsAnObjectAndAnsweredInChunks(t *testing.T) {
|
|
ctx := context.Background()
|
|
objs := &objectsMem{objs: map[string][]byte{}}
|
|
ix := &Indexer{Module: testSelf, Inputs: newMem(), InputObjects: objs, Now: time.Now}
|
|
body := make([]byte, 3<<20+17)
|
|
if _, err := rand.Read(body); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
digest, err := ix.KeepInput(ctx, "ask-big", body)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(objs.objs) != 1 {
|
|
t.Fatalf("a large input is one object beside the state, got %d", len(objs.objs))
|
|
}
|
|
if got, err := InputBody(ctx, ix.Inputs, objs, digest); err != nil || !bytes.Equal(got, body) {
|
|
t.Fatalf("the input read back: %d bytes, %v", len(got), err)
|
|
}
|
|
line := &lineMem{chunks: map[string][]byte{}}
|
|
verb := InputVerb(ix.Inputs, objs, line, time.Second, t.Logf)
|
|
raw, _ := json.Marshal(meshcatalog.InputInput{Digest: digest, Call: "0123456789abcdef"})
|
|
out, err := verb("", raw)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ans := out.(meshcatalog.InputAnswer)
|
|
want := int64((len(body) + buscalls.SeatChunkBytes - 1) / buscalls.SeatChunkBytes)
|
|
if ans.Digest != digest || ans.Chunks != want || want < 6 {
|
|
t.Fatalf("the answer %+v, want %d chunks", ans, want)
|
|
}
|
|
send := buscalls.SeatChunks{Seat: meshcatalog.Seat, Verb: meshcatalog.VerbInput, Call: "0123456789abcdef"}
|
|
var got []byte
|
|
deadline := time.Now().Add(5 * time.Second)
|
|
for n := int64(1); n <= want; n++ {
|
|
for {
|
|
line.mu.Lock()
|
|
data, ok := line.chunks[send.Chunk(n)]
|
|
line.mu.Unlock()
|
|
if ok {
|
|
var c buscalls.SeatChunk
|
|
if err := json.Unmarshal(data, &c); err != nil || c.N != n || c.Of != want {
|
|
t.Fatalf("chunk %d: %+v, %v", n, c, err)
|
|
}
|
|
got = append(got, c.Data...)
|
|
break
|
|
}
|
|
if time.Now().After(deadline) {
|
|
t.Fatalf("chunk %d of %d never sent", n, want)
|
|
}
|
|
time.Sleep(time.Millisecond)
|
|
}
|
|
}
|
|
if !bytes.Equal(got, body) {
|
|
t.Error("the chunks do not carry the input's bytes")
|
|
}
|
|
if _, err := verb("", json.RawMessage(`{"digest":"sha256:none","call":"0123456789abcdef"}`)); err == nil {
|
|
t.Error("an input never kept is answered")
|
|
}
|
|
if _, err := verb("", json.RawMessage(`{"digest":"`+digest+`","call":"x"}`)); err == nil {
|
|
t.Error("a call of no token is answered")
|
|
}
|
|
|
|
// the bucket holds only inputs still asked: the ended ask's object goes with its key, and one a past delete left goes at the next write
|
|
if err := ix.HearInputEnded(ctx, nodebuildgo.StepEnded{Ask: "ask-big"}); err != nil || len(objs.objs) != 0 {
|
|
t.Fatalf("an ended ask's input leaves its object: %d left, %v", len(objs.objs), err)
|
|
}
|
|
objs.objs["inputs.orphan"] = make([]byte, 100)
|
|
objs.cap = len(body)*4/3 + 400 // the base64 text, and the orphan below it
|
|
if _, err := ix.KeepInput(ctx, "ask-two", body); err != nil || len(objs.objs) != 1 {
|
|
t.Fatalf("a write first removes the objects no key names: %d objects, %v", len(objs.objs), err)
|
|
}
|
|
// a full bucket is said loudly, naming the bucket and its fill
|
|
_, err = ix.KeepInput(ctx, "ask-three", append([]byte{1}, body...))
|
|
if err == nil || !strings.Contains(err.Error(), "inputs-objects") || !strings.Contains(err.Error(), "1 objects") {
|
|
t.Fatalf("a refused write names the bucket and its fill: %v", err)
|
|
}
|
|
}
|