Files
mesh-delivery/modules/mesh-delivery-indexer/internal/index/run_input_chunks_test.go
T
jschoubben ac0befa9b9 Indexer: an input leaves nothing in its object store; a full store is said by name and fill
The indexer's inputs-objects bucket filled twice: deleting an input's key (step-ended, the day sweep)
left its object, which only the person's drop tool removed. The input's key and object now go together,
each write first drops the objects no key names, and a write the store refuses names the store and its fill.

Builds: design 47 G §354, §712, A-237
2026-11-16 15:31:26 +01:00

175 lines
5.6 KiB
Go

package index
import (
"bytes"
"context"
"crypto/rand"
"encoding/json"
"errors"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"git.novox.be/novox/mesh-sdk/go/stateobjects"
)
type objectsMem struct {
mu sync.Mutex
objs map[string][]byte
cap int // the most bytes the bucket holds; zero for no limit (a full bucket refuses: maximum bytes exceeded)
}
func (*objectsMem) MostBytes() int { return 0 }
func (o *objectsMem) Put(name string, data []byte) error {
o.mu.Lock()
defer o.mu.Unlock()
if o.cap > 0 {
used := len(data)
for _, d := range o.objs {
used += len(d)
}
if used > o.cap {
return errors.New("nats: API error: code=503 err_code=10077 description=maximum bytes exceeded")
}
}
o.objs[name] = data
return nil
}
func (o *objectsMem) Get(name string) ([]byte, error) {
o.mu.Lock()
defer o.mu.Unlock()
d, ok := o.objs[name]
if !ok {
return nil, errors.New("no such object")
}
return d, nil
}
func (o *objectsMem) Delete(name string) error {
o.mu.Lock()
defer o.mu.Unlock()
delete(o.objs, name)
return nil
}
func (o *objectsMem) List() ([]stateobjects.Object, error) {
o.mu.Lock()
defer o.mu.Unlock()
var out []stateobjects.Object
for n, d := range o.objs {
out = append(out, stateobjects.Object{Name: n, Size: int64(len(d))})
}
return out, nil
}
// lineMem is a core bus in memory: a chunk published is kept, and its acknowledgement heard at once.
type lineMem struct {
mu sync.Mutex
hear nats.MsgHandler
chunks map[string][]byte
}
func (l *lineMem) Publish(subject string, data []byte) error {
l.mu.Lock()
l.chunks[subject] = data
h := l.hear
l.mu.Unlock()
var c buscalls.SeatChunk
if err := json.Unmarshal(data, &c); err != nil {
return err
}
go h(&nats.Msg{Subject: buscalls.SeatChunks{Seat: meshcatalog.Seat, Verb: meshcatalog.VerbInput, Call: "0123456789abcdef"}.Ack(c.N)})
return nil
}
func (l *lineMem) Subscribe(_ string, fn nats.MsgHandler) (*nats.Subscription, error) {
l.mu.Lock()
l.hear = fn
l.mu.Unlock()
return nil, nil
}
// Design 47 G §712, A-237: an input is of any size: over a key's limit it is kept as an object beside the state, and the verb
// input answers it in chunks of at most 512 KiB, each acknowledged before the next.
func TestAnInputOverOneMiBIsKeptAsAnObjectAndAnsweredInChunks(t *testing.T) {
ctx := context.Background()
objs := &objectsMem{objs: map[string][]byte{}}
ix := &Indexer{Module: testSelf, Inputs: newMem(), InputObjects: objs, Now: time.Now}
body := make([]byte, 3<<20+17)
if _, err := rand.Read(body); err != nil {
t.Fatal(err)
}
digest, err := ix.KeepInput(ctx, "ask-big", body)
if err != nil {
t.Fatal(err)
}
if len(objs.objs) != 1 {
t.Fatalf("a large input is one object beside the state, got %d", len(objs.objs))
}
if got, err := InputBody(ctx, ix.Inputs, objs, digest); err != nil || !bytes.Equal(got, body) {
t.Fatalf("the input read back: %d bytes, %v", len(got), err)
}
line := &lineMem{chunks: map[string][]byte{}}
verb := InputVerb(ix.Inputs, objs, line, time.Second, t.Logf)
raw, _ := json.Marshal(meshcatalog.InputInput{Digest: digest, Call: "0123456789abcdef"})
out, err := verb("", raw)
if err != nil {
t.Fatal(err)
}
ans := out.(meshcatalog.InputAnswer)
want := int64((len(body) + buscalls.SeatChunkBytes - 1) / buscalls.SeatChunkBytes)
if ans.Digest != digest || ans.Chunks != want || want < 6 {
t.Fatalf("the answer %+v, want %d chunks", ans, want)
}
send := buscalls.SeatChunks{Seat: meshcatalog.Seat, Verb: meshcatalog.VerbInput, Call: "0123456789abcdef"}
var got []byte
deadline := time.Now().Add(5 * time.Second)
for n := int64(1); n <= want; n++ {
for {
line.mu.Lock()
data, ok := line.chunks[send.Chunk(n)]
line.mu.Unlock()
if ok {
var c buscalls.SeatChunk
if err := json.Unmarshal(data, &c); err != nil || c.N != n || c.Of != want {
t.Fatalf("chunk %d: %+v, %v", n, c, err)
}
got = append(got, c.Data...)
break
}
if time.Now().After(deadline) {
t.Fatalf("chunk %d of %d never sent", n, want)
}
time.Sleep(time.Millisecond)
}
}
if !bytes.Equal(got, body) {
t.Error("the chunks do not carry the input's bytes")
}
if _, err := verb("", json.RawMessage(`{"digest":"sha256:none","call":"0123456789abcdef"}`)); err == nil {
t.Error("an input never kept is answered")
}
if _, err := verb("", json.RawMessage(`{"digest":"`+digest+`","call":"x"}`)); err == nil {
t.Error("a call of no token is answered")
}
// the bucket holds only inputs still asked: the ended ask's object goes with its key, and one a past delete left goes at the next write
if err := ix.HearInputEnded(ctx, nodebuildgo.StepEnded{Ask: "ask-big"}); err != nil || len(objs.objs) != 0 {
t.Fatalf("an ended ask's input leaves its object: %d left, %v", len(objs.objs), err)
}
objs.objs["inputs.orphan"] = make([]byte, 100)
objs.cap = len(body)*4/3 + 400 // the base64 text, and the orphan below it
if _, err := ix.KeepInput(ctx, "ask-two", body); err != nil || len(objs.objs) != 1 {
t.Fatalf("a write first removes the objects no key names: %d objects, %v", len(objs.objs), err)
}
// a full bucket is said loudly, naming the bucket and its fill
_, err = ix.KeepInput(ctx, "ask-three", append([]byte{1}, body...))
if err == nil || !strings.Contains(err.Error(), "inputs-objects") || !strings.Contains(err.Error(), "1 objects") {
t.Fatalf("a refused write names the bucket and its fill: %v", err)
}
}