mesh-delivery/modules/mesh-delivery-indexer/internal/index/run_input_chunks_test.go

package index

import (
	"bytes"
	"context"
	"crypto/rand"
	"encoding/json"
	"errors"
	"sync"
	"testing"
	"time"

	"github.com/nats-io/nats.go"

	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
	"git.novox.be/novox/mesh-sdk/go/buscalls"
	"git.novox.be/novox/mesh-sdk/go/stateobjects"
)

type objectsMem struct {
	mu   sync.Mutex
	objs map[string][]byte
}

func (*objectsMem) MostBytes() int { return 0 }
func (o *objectsMem) Put(name string, data []byte) error {
	o.mu.Lock()
	defer o.mu.Unlock()
	o.objs[name] = data
	return nil
}
func (o *objectsMem) Get(name string) ([]byte, error) {
	o.mu.Lock()
	defer o.mu.Unlock()
	d, ok := o.objs[name]
	if !ok {
		return nil, errors.New("no such object")
	}
	return d, nil
}
func (o *objectsMem) Delete(name string) error             { return nil }
func (o *objectsMem) List() ([]stateobjects.Object, error) { return nil, nil }

// lineMem is a core bus in memory: a chunk published is kept, and its acknowledgement heard at once.
type lineMem struct {
	mu     sync.Mutex
	hear   nats.MsgHandler
	chunks map[string][]byte
}

func (l *lineMem) Publish(subject string, data []byte) error {
	l.mu.Lock()
	l.chunks[subject] = data
	h := l.hear
	l.mu.Unlock()
	var c buscalls.SeatChunk
	if err := json.Unmarshal(data, &c); err != nil {
		return err
	}
	go h(&nats.Msg{Subject: buscalls.SeatChunks{Seat: meshcatalog.Seat, Verb: meshcatalog.VerbInput, Call: "0123456789abcdef"}.Ack(c.N)})
	return nil
}

func (l *lineMem) Subscribe(_ string, fn nats.MsgHandler) (*nats.Subscription, error) {
	l.mu.Lock()
	l.hear = fn
	l.mu.Unlock()
	return nil, nil
}

// Design 47 G §712, A-237: an input is of any size: over a key's limit it is kept as an object beside the state, and the verb
// input answers it in chunks of at most 512 KiB, each acknowledged before the next.
func TestAnInputOverOneMiBIsKeptAsAnObjectAndAnsweredInChunks(t *testing.T) {
	ctx := context.Background()
	objs := &objectsMem{objs: map[string][]byte{}}
	ix := &Indexer{Inputs: newMem(), InputObjects: objs, Now: time.Now}
	body := make([]byte, 3<<20+17)
	if _, err := rand.Read(body); err != nil {
		t.Fatal(err)
	}
	digest, err := ix.KeepInput(ctx, "ask-big", body)
	if err != nil {
		t.Fatal(err)
	}
	if len(objs.objs) != 1 {
		t.Fatalf("a large input is one object beside the state, got %d", len(objs.objs))
	}
	if got, err := InputBody(ctx, ix.Inputs, objs, digest); err != nil || !bytes.Equal(got, body) {
		t.Fatalf("the input read back: %d bytes, %v", len(got), err)
	}
	line := &lineMem{chunks: map[string][]byte{}}
	verb := InputVerb(ix.Inputs, objs, line, time.Second, t.Logf)
	raw, _ := json.Marshal(meshcatalog.InputInput{Digest: digest, Call: "0123456789abcdef"})
	out, err := verb("", raw)
	if err != nil {
		t.Fatal(err)
	}
	ans := out.(meshcatalog.InputAnswer)
	want := int64((len(body) + buscalls.SeatChunkBytes - 1) / buscalls.SeatChunkBytes)
	if ans.Digest != digest || ans.Chunks != want || want < 6 {
		t.Fatalf("the answer %+v, want %d chunks", ans, want)
	}
	send := buscalls.SeatChunks{Seat: meshcatalog.Seat, Verb: meshcatalog.VerbInput, Call: "0123456789abcdef"}
	var got []byte
	deadline := time.Now().Add(5 * time.Second)
	for n := int64(1); n <= want; n++ {
		for {
			line.mu.Lock()
			data, ok := line.chunks[send.Chunk(n)]
			line.mu.Unlock()
			if ok {
				var c buscalls.SeatChunk
				if err := json.Unmarshal(data, &c); err != nil || c.N != n || c.Of != want {
					t.Fatalf("chunk %d: %+v, %v", n, c, err)
				}
				got = append(got, c.Data...)
				break
			}
			if time.Now().After(deadline) {
				t.Fatalf("chunk %d of %d never sent", n, want)
			}
			time.Sleep(time.Millisecond)
		}
	}
	if !bytes.Equal(got, body) {
		t.Error("the chunks do not carry the input's bytes")
	}
	if _, err := verb("", json.RawMessage(`{"digest":"sha256:none","call":"0123456789abcdef"}`)); err == nil {
		t.Error("an input never kept is answered")
	}
	if _, err := verb("", json.RawMessage(`{"digest":"`+digest+`","call":"x"}`)); err == nil {
		t.Error("a call of no token is answered")
	}
}