Files
mesh-delivery/modules/mesh-delivery-indexer/internal/index/run_input_chunks_test.go
T
jschoubben 0fffea89b1 B1 (indexer): the indexer's own name is its credential's module, no constant
index.Module is gone: the indexer and its registry carry the credential's module (findings' observer and ids, its own
entry and graph key, the history subject, the views emitter and reader), the tools take the served-as module.

Builds: design 47 G A-296, §428

Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
2026-11-12 23:37:54 +01:00

135 lines
4.0 KiB
Go

package index
import (
"bytes"
"context"
"crypto/rand"
"encoding/json"
"errors"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"git.novox.be/novox/mesh-sdk/go/stateobjects"
)
type objectsMem struct {
mu sync.Mutex
objs map[string][]byte
}
func (*objectsMem) MostBytes() int { return 0 }
func (o *objectsMem) Put(name string, data []byte) error {
o.mu.Lock()
defer o.mu.Unlock()
o.objs[name] = data
return nil
}
func (o *objectsMem) Get(name string) ([]byte, error) {
o.mu.Lock()
defer o.mu.Unlock()
d, ok := o.objs[name]
if !ok {
return nil, errors.New("no such object")
}
return d, nil
}
func (o *objectsMem) Delete(name string) error { return nil }
func (o *objectsMem) List() ([]stateobjects.Object, error) { return nil, nil }
// lineMem is a core bus in memory: a chunk published is kept, and its acknowledgement heard at once.
type lineMem struct {
mu sync.Mutex
hear nats.MsgHandler
chunks map[string][]byte
}
func (l *lineMem) Publish(subject string, data []byte) error {
l.mu.Lock()
l.chunks[subject] = data
h := l.hear
l.mu.Unlock()
var c buscalls.SeatChunk
if err := json.Unmarshal(data, &c); err != nil {
return err
}
go h(&nats.Msg{Subject: buscalls.SeatChunks{Seat: meshcatalog.Seat, Verb: meshcatalog.VerbInput, Call: "0123456789abcdef"}.Ack(c.N)})
return nil
}
func (l *lineMem) Subscribe(_ string, fn nats.MsgHandler) (*nats.Subscription, error) {
l.mu.Lock()
l.hear = fn
l.mu.Unlock()
return nil, nil
}
// Design 47 G §712, A-237: an input is of any size: over a key's limit it is kept as an object beside the state, and the verb
// input answers it in chunks of at most 512 KiB, each acknowledged before the next.
func TestAnInputOverOneMiBIsKeptAsAnObjectAndAnsweredInChunks(t *testing.T) {
ctx := context.Background()
objs := &objectsMem{objs: map[string][]byte{}}
ix := &Indexer{Module: testSelf, Inputs: newMem(), InputObjects: objs, Now: time.Now}
body := make([]byte, 3<<20+17)
if _, err := rand.Read(body); err != nil {
t.Fatal(err)
}
digest, err := ix.KeepInput(ctx, "ask-big", body)
if err != nil {
t.Fatal(err)
}
if len(objs.objs) != 1 {
t.Fatalf("a large input is one object beside the state, got %d", len(objs.objs))
}
if got, err := InputBody(ctx, ix.Inputs, objs, digest); err != nil || !bytes.Equal(got, body) {
t.Fatalf("the input read back: %d bytes, %v", len(got), err)
}
line := &lineMem{chunks: map[string][]byte{}}
verb := InputVerb(ix.Inputs, objs, line, time.Second, t.Logf)
raw, _ := json.Marshal(meshcatalog.InputInput{Digest: digest, Call: "0123456789abcdef"})
out, err := verb("", raw)
if err != nil {
t.Fatal(err)
}
ans := out.(meshcatalog.InputAnswer)
want := int64((len(body) + buscalls.SeatChunkBytes - 1) / buscalls.SeatChunkBytes)
if ans.Digest != digest || ans.Chunks != want || want < 6 {
t.Fatalf("the answer %+v, want %d chunks", ans, want)
}
send := buscalls.SeatChunks{Seat: meshcatalog.Seat, Verb: meshcatalog.VerbInput, Call: "0123456789abcdef"}
var got []byte
deadline := time.Now().Add(5 * time.Second)
for n := int64(1); n <= want; n++ {
for {
line.mu.Lock()
data, ok := line.chunks[send.Chunk(n)]
line.mu.Unlock()
if ok {
var c buscalls.SeatChunk
if err := json.Unmarshal(data, &c); err != nil || c.N != n || c.Of != want {
t.Fatalf("chunk %d: %+v, %v", n, c, err)
}
got = append(got, c.Data...)
break
}
if time.Now().After(deadline) {
t.Fatalf("chunk %d of %d never sent", n, want)
}
time.Sleep(time.Millisecond)
}
}
if !bytes.Equal(got, body) {
t.Error("the chunks do not carry the input's bytes")
}
if _, err := verb("", json.RawMessage(`{"digest":"sha256:none","call":"0123456789abcdef"}`)); err == nil {
t.Error("an input never kept is answered")
}
if _, err := verb("", json.RawMessage(`{"digest":"`+digest+`","call":"x"}`)); err == nil {
t.Error("a call of no token is answered")
}
}