Files
mesh-delivery/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/inputs_test.go
T
jschoubben 7697ef5988 assigner: a run's bound holds its ask, armed by compare-and-set, and its end comes before the input's delete (A-470n, A-470o, A-470p)
A bound was armed by a delete and a create with no revision, holding nothing, so a late ask-taken or
step-progress of a replaced run armed a clock that later ended the run asked next, and an expiry heard
after the bound was armed again ended a run that kept saying progress. A bound key now holds its run's
ask, published at the key's latest revision with its own time to live, and is deleted at its own
revision when the run read again no longer names that ask; a late signal moves nothing and is
journaled; an expiry ends the run only while its bound is still absent when read again and the run is
taken (bound.) or untaken (untaken.), the untaken expiry's readers said superseded as the key is gone.
A step-ended's run end is written before the kept input is deleted; a failed delete asks the event
again, whose redelivery finds the run ended and still deletes the input.

Builds: design 47 G §720, §719; A-470n, A-470o, A-470p
2026-11-17 12:39:12 +01:00

504 lines
17 KiB
Go

package main
import (
"bytes"
"crypto/rand"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"git.novox.be/novox/mesh-sdk/go/stateobjects"
)
type kvMem struct {
mu sync.Mutex
vals map[string][]byte
revs map[string]uint64
// delErr, when set, fails the next Delete once.
delErr error
}
func newKVMem() *kvMem { return &kvMem{vals: map[string][]byte{}, revs: map[string]uint64{}} }
func (k *kvMem) Get(key string) (*stateobjects.Entry, error) {
k.mu.Lock()
defer k.mu.Unlock()
v, ok := k.vals[key]
if !ok {
return nil, nil
}
return &stateobjects.Entry{Value: v, Revision: k.revs[key]}, nil
}
func (k *kvMem) Create(key string, v []byte) error {
k.mu.Lock()
defer k.mu.Unlock()
if _, ok := k.vals[key]; ok {
return stateobjects.ErrChanged
}
k.vals[key], k.revs[key] = v, 1
return nil
}
func (k *kvMem) Update(key string, v []byte, rev uint64) error {
k.mu.Lock()
defer k.mu.Unlock()
if k.revs[key] != rev {
return stateobjects.ErrChanged
}
k.vals[key], k.revs[key] = v, rev+1
return nil
}
func (k *kvMem) Delete(key string) error {
k.mu.Lock()
defer k.mu.Unlock()
if err := k.delErr; err != nil {
k.delErr = nil
return err
}
delete(k.vals, key)
return nil
}
func (k *kvMem) Keys() ([]string, error) {
k.mu.Lock()
defer k.mu.Unlock()
var out []string
for key := range k.vals {
out = append(out, key)
}
sort.Strings(out)
return out, nil
}
type objectsMem struct {
mu sync.Mutex
objs map[string][]byte
listErr error // List fails with it
}
func (*objectsMem) MostBytes() int { return 0 }
func (o *objectsMem) Put(name string, data []byte) error {
o.mu.Lock()
defer o.mu.Unlock()
o.objs[name] = data
return nil
}
func (o *objectsMem) Get(name string) ([]byte, error) {
o.mu.Lock()
defer o.mu.Unlock()
d, ok := o.objs[name]
if !ok {
return nil, errors.New("no such object")
}
return d, nil
}
func (o *objectsMem) Delete(name string) error {
o.mu.Lock()
defer o.mu.Unlock()
delete(o.objs, name)
return nil
}
func (o *objectsMem) List() ([]stateobjects.Object, error) {
o.mu.Lock()
defer o.mu.Unlock()
if o.listErr != nil {
return nil, o.listErr
}
var out []stateobjects.Object
for n, d := range o.objs {
out = append(out, stateobjects.Object{Name: n, Size: int64(len(d))})
}
return out, nil
}
// lineMem is a core bus in memory: a chunk published is kept, and its acknowledgement heard at once.
type lineMem struct {
mu sync.Mutex
hear nats.MsgHandler
chunks map[string][]byte
}
func (l *lineMem) Publish(subject string, data []byte) error {
l.mu.Lock()
l.chunks[subject] = data
h := l.hear
l.mu.Unlock()
var c buscalls.SeatChunk
if err := json.Unmarshal(data, &c); err != nil {
return err
}
go h(&nats.Msg{Subject: buscalls.SeatChunks{Seat: meshwants.Seat, Verb: meshwants.VerbInput, Call: "0123456789abcdef"}.Ack(c.N)})
return nil
}
func (l *lineMem) Subscribe(_ string, fn nats.MsgHandler) (*nats.Subscription, error) {
l.mu.Lock()
l.hear = fn
l.mu.Unlock()
return nil, nil
}
func testKeeper(objs *objectsMem, now func() time.Time) *inputKeeper {
return &inputKeeper{vals: stateobjects.Values{KV: newKVMem(), Objects: objs, Store: "inputs-objects"}, now: now, logf: func(string, ...any) {}}
}
// Design 47 G §354, §712, A-237: an input of any size is kept as an object beside the state, and the seat mesh-wants' verb input
// answers it in chunks of at most 512 KiB, each acknowledged before the next.
func TestAnInputOverOneMiBIsKeptAsAnObjectAndAnsweredInChunks(t *testing.T) {
objs := &objectsMem{objs: map[string][]byte{}}
k := testKeeper(objs, time.Now)
body := make([]byte, 3<<20+17)
if _, err := rand.Read(body); err != nil {
t.Fatal(err)
}
digest, err := k.keep("ask-big", body)
if err != nil {
t.Fatal(err)
}
if len(objs.objs) != 1 {
t.Fatalf("a large input is one object beside the state, got %d", len(objs.objs))
}
if got, err := k.body(digest); err != nil || !bytes.Equal(got, body) {
t.Fatalf("the input read back: %d bytes, %v", len(got), err)
}
line := &lineMem{chunks: map[string][]byte{}}
verb := k.verb(line, time.Second)
raw, _ := json.Marshal(meshwants.InputInput{Digest: digest, Call: "0123456789abcdef"})
out, err := verb("", raw)
if err != nil {
t.Fatal(err)
}
ans := out.(meshwants.InputAnswer)
want := int64((len(body) + buscalls.SeatChunkBytes - 1) / buscalls.SeatChunkBytes)
if ans.Digest != digest || ans.Chunks != want || want < 6 {
t.Fatalf("the answer %+v, want %d chunks", ans, want)
}
send := buscalls.SeatChunks{Seat: meshwants.Seat, Verb: meshwants.VerbInput, Call: "0123456789abcdef"}
var got []byte
deadline := time.Now().Add(5 * time.Second)
for n := int64(1); n <= want; n++ {
for {
line.mu.Lock()
data, ok := line.chunks[send.Chunk(n)]
line.mu.Unlock()
if ok {
var c buscalls.SeatChunk
if err := json.Unmarshal(data, &c); err != nil || c.N != n || c.Of != want {
t.Fatalf("chunk %d: %+v, %v", n, c, err)
}
got = append(got, c.Data...)
break
}
if time.Now().After(deadline) {
t.Fatalf("chunk %d of %d never sent", n, want)
}
time.Sleep(time.Millisecond)
}
}
if !bytes.Equal(got, body) {
t.Error("the chunks do not carry the input's bytes")
}
if _, err := verb("", json.RawMessage(`{"digest":"`+strings.Repeat("b", 64)+`","call":"0123456789abcdef"}`)); err == nil {
t.Error("an input never kept is answered")
}
if _, err := verb("", json.RawMessage(`{"digest":"`+digest+`","call":"x"}`)); err == nil {
t.Error("a call of no token is answered")
}
// assigner_inputs lists it with its size and gives one whole by digest
list, err := inputsRead(k).Run(map[string]any{})
if err != nil || len(list.(map[string]any)["inputs"].([]inputSaid)) != 1 || list.(map[string]any)["inputs"].([]inputSaid)[0].Size != len(body) {
t.Errorf("assigner_inputs = %+v %v", list, err)
}
}
// §354: an input is deleted when the build seat's step-ended of its ask is heard, or a day after its write, judged at each new ask.
func TestAnInputIsDeletedAtItsStepEndedOrAfterADay(t *testing.T) {
now := time.Date(2026, 11, 12, 10, 0, 0, 0, time.UTC)
k := testKeeper(&objectsMem{objs: map[string][]byte{}}, func() time.Time { return now })
d1, err := k.keep("ask/one", []byte(`{"heads":"abc"}`))
if err != nil {
t.Fatal(err)
}
if _, err := k.body(d1); err != nil {
t.Fatalf("a kept input is not read: %v", err)
}
if err := k.ended("ask/other"); err != nil {
t.Fatal(err)
}
if _, err := k.body(d1); err != nil {
t.Fatalf("the step-ended of another ask deleted it: %v", err)
}
if err := k.ended("ask/one"); err != nil {
t.Fatal(err)
}
if _, err := k.body(d1); err == nil {
t.Error("the input of an ended ask is still answered")
}
if keys, _ := k.vals.KV.Keys(); len(keys) != 0 {
t.Errorf("keys left after the step-ended: %v", keys)
}
d2, _ := k.keep("ask/two", []byte(`{"heads":"def"}`))
now = now.Add(25 * time.Hour)
if _, err := k.keep("ask/three", []byte(`{"heads":"ghi"}`)); err != nil {
t.Fatal(err)
}
if _, err := k.body(d2); err == nil {
t.Error("an input a day old is still kept at the next ask")
}
// A-470o, A-470p: the step-ended's run end is written before the input is deleted; a delete that fails asks the event again,
// whose redelivery finds the run ended and still deletes the input; a run end not written asks it again with the input kept.
for _, c := range []struct {
name string
delErr error
runErr error
naked bool
runs int
deleted bool
}{
{name: "the run's end before the input's delete", runs: 1, deleted: true},
{name: "an input not deleted asks the event again", delErr: errors.New("the bus is away"), naked: true, runs: 2, deleted: true},
{name: "a run's end not written asks the event again", runErr: errors.New("the run is not read"), naked: true, runs: 1},
} {
t.Run(c.name, func(t *testing.T) {
k := testKeeper(&objectsMem{objs: map[string][]byte{}}, time.Now)
d, _ := k.keep("ask/run", []byte(`{"heads":"abc"}`))
k.vals.KV.(*kvMem).delErr = c.delErr
runs := 0
runEnded := func(e buildstep.StepEnded) error {
runs++
if _, err := k.body(d); err != nil {
t.Errorf("the run's end is written after the input's delete (%d)", runs)
}
return c.runErr
}
raw, _ := json.Marshal(buildstep.StepEnded{Ask: "ask/run"})
m := &heardMsg{data: raw, subject: "step-ended"}
heardStepEnded(k, runEnded, m, t.Logf)
if m.naked != c.naked || m.acked == c.naked {
t.Fatalf("acked %v, naked %v, want naked %v", m.acked, m.naked, c.naked)
}
if m.naked && c.delErr != nil { // the redelivery
m = &heardMsg{data: raw, subject: "step-ended"}
heardStepEnded(k, func(e buildstep.StepEnded) error { runs++; return nil }, m, t.Logf)
}
if _, err := k.body(d); runs != c.runs || (err != nil) != c.deleted {
t.Errorf("the run's end heard %d times (want %d), the input deleted %v (want %v)", runs, c.runs, err != nil, c.deleted)
}
})
}
}
// Design 47 G A-465: an input the assigner ends itself (the step-ended of its ask, the sweep of a day-old key) ends its object
// too; the objects of keys another part deleted are removed by dropKeyGone and by the tool assigner_inputs_drop, which lists
// them as key-gone first, and never removes an object a key names.
func TestAnEndedInputEndsItsObjectAndTheDropToolRemovesWhatAKeyLeft(t *testing.T) {
now := time.Date(2026, 11, 17, 10, 0, 0, 0, time.UTC)
objs := &objectsMem{objs: map[string][]byte{}}
k := testKeeper(objs, func() time.Time { return now })
big := func(b byte) []byte { return bytes.Repeat([]byte{b}, 2<<20) }
if _, err := k.keep("ask-one", big('a')); err != nil {
t.Fatal(err)
}
if _, err := k.keep("ask-two", big('b')); err != nil {
t.Fatal(err)
}
if len(objs.objs) != 2 {
t.Fatalf("objects = %d, want 2", len(objs.objs))
}
if err := k.ended("ask-one"); err != nil || len(objs.objs) != 1 {
t.Fatalf("the step-ended left %d objects (%v), want 1", len(objs.objs), err)
}
now = now.Add(25 * time.Hour)
if _, err := k.keep("ask-three", big('c')); err != nil {
t.Fatal(err)
}
if len(objs.objs) != 1 {
t.Fatalf("after the sweep of the day-old input %d objects, want only the new one", len(objs.objs))
}
// An object whose key another part deleted is listed as key-gone and removed by the tool; a live one stays.
keys, _ := k.vals.KV.Keys()
for _, key := range keys {
if !strings.HasPrefix(key, inputsAskPrefix) {
if err := k.vals.KV.Delete(key); err != nil {
t.Fatal(err)
}
}
}
live := []byte(`{"heads":"x"}`)
if _, err := k.keep("ask-four", append(live, make([]byte, 2<<20)...)); err != nil {
t.Fatal(err)
}
list, err := inputsRead(k).Run(map[string]any{})
if err != nil || len(list.(map[string]any)["key-gone"].([]map[string]any)) != 1 {
t.Fatalf("assigner_inputs key-gone = %+v %v, want the one object of the deleted key", list, err)
}
ask := func(string, any) (json.RawMessage, error) {
return json.RawMessage(`{"answer":{"log":"hand-acts"}}`), nil
}
if _, err := inputsDrop(k, ask).Run(map[string]any{}); err == nil {
t.Error("dropped with no reason")
}
out, err := inputsDrop(k, ask).Run(map[string]any{"why": "a key left its object"})
if err != nil || out.(map[string]any)["count"] != 1 || len(objs.objs) != 1 {
t.Fatalf("assigner_inputs_drop = %+v %v, %d objects left: want one removed and the live one kept", out, err, len(objs.objs))
}
// The start drops what a key left, said with its count and bytes.
if _, err := k.keep("ask-five", live); err != nil {
t.Fatal(err)
}
for _, key := range keysOf(t, k) {
if !strings.HasPrefix(key, inputsAskPrefix) {
_ = k.vals.KV.Delete(key)
}
}
var logged []string
k.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
before := len(objs.objs)
k.dropAtStart()
if len(objs.objs) != 0 || before == 0 || len(logged) != 1 || !strings.Contains(logged[0], fmt.Sprintf("%d objects of deleted keys removed at the start", before)) || !strings.Contains(logged[0], " bytes") {
t.Errorf("the start left %d of %d objects and said %q: want every key-gone object removed and its count and bytes said", len(objs.objs), before, logged)
}
}
func keysOf(t *testing.T, k *inputKeeper) []string {
t.Helper()
keys, err := k.vals.KV.Keys()
if err != nil {
t.Fatal(err)
}
return keys
}
// Design 47 G A-465: assigner_inputs says why key-gone objects are not known when the objects cannot be listed, and never
// answers them as none.
func TestAssignerInputsSaysWhenTheKeyGoneObjectsCannotBeListed(t *testing.T) {
objs := &objectsMem{objs: map[string][]byte{}, listErr: errors.New("the object store does not answer")}
k := testKeeper(objs, time.Now)
out, err := inputsRead(k).Run(map[string]any{})
if err != nil {
t.Fatal(err)
}
m := out.(map[string]any)
if msg, _ := m["key-gone-error"].(string); !strings.Contains(msg, "does not answer") {
t.Errorf("answer %+v: want key-gone-error naming the failure", m)
}
if _, has := m["key-gone"]; has {
t.Errorf("answer %+v: an unreadable store is not answered as no key-gone objects", m)
}
}
// Design 47 G A-465a: an input is kept once per content under its sha256 alone; every ask names the digest it uses, and the input
// goes, with its object, at the step-ended of the last ask that names it, or a day after its last write; a request the cap still
// refuses says the fill; an input of the old form (a key that is no digest) is ended with its object.
func TestAnInputIsKeptOncePerContentAndEndsWithItsLastAsk(t *testing.T) {
now := time.Date(2026, 11, 17, 10, 0, 0, 0, time.UTC)
objs := &objectsMem{objs: map[string][]byte{}}
k := testKeeper(objs, func() time.Time { return now })
body := bytes.Repeat([]byte{'n'}, 2<<20)
var digests []string
for _, ask := range []string{"ask-1", "ask-2", "ask-3"} {
d, err := k.keep(ask, body)
if err != nil {
t.Fatal(err)
}
digests = append(digests, d)
}
if digests[0] != digestOf(body) || digests[1] != digests[0] || digests[2] != digests[0] {
t.Fatalf("digests = %v, want the sha256 of the bytes each time", digests)
}
if _, err := k.vals.KV.Get(digests[0]); err != nil {
t.Fatal(err)
}
if e, _ := k.vals.KV.Get(digests[0]); e == nil || len(objs.objs) != 1 {
t.Fatalf("one key by the digest and one object wanted, got entry %v and %d objects", e, len(objs.objs))
}
for _, key := range keysOf(t, k) {
if !strings.HasPrefix(key, inputsAskPrefix) && key != digests[0] {
t.Errorf("a key %q beside the one input", key)
}
}
if e, _ := k.vals.KV.Get(inputsAskPrefix + "ask-2"); e == nil || !strings.Contains(string(e.Value), digests[0]) || !strings.Contains(string(e.Value), "20261117T100000Z") {
t.Errorf("the ask does not name the digest it uses: %v", e)
}
for _, ask := range []string{"ask-1", "ask-2"} {
if err := k.ended(ask); err != nil {
t.Fatal(err)
}
if _, err := k.body(digests[0]); err != nil {
t.Fatalf("the input went at the end of %s while ask-3 still names it: %v", ask, err)
}
}
if err := k.ended("ask-3"); err != nil {
t.Fatal(err)
}
if _, err := k.body(digests[0]); err == nil || len(objs.objs) != 0 || len(keysOf(t, k)) != 0 {
t.Errorf("the last step-ended left the input (%v), %d objects, keys %v", err, len(objs.objs), keysOf(t, k))
}
// a day after its last write, a renewed write counted
if _, err := k.keep("ask-4", body); err != nil {
t.Fatal(err)
}
now = now.Add(20 * time.Hour)
if _, err := k.keep("ask-5", body); err != nil {
t.Fatal(err)
}
now = now.Add(20 * time.Hour)
if _, err := k.keep("ask-6", []byte("other")); err != nil {
t.Fatal(err)
}
if _, err := k.body(digests[0]); err != nil {
t.Errorf("an input written again 20 hours ago was swept: %v", err)
}
now = now.Add(25 * time.Hour)
if _, err := k.keep("ask-7", []byte("another")); err != nil {
t.Fatal(err)
}
if _, err := k.body(digests[0]); err == nil || len(objs.objs) != 0 {
t.Errorf("an input a day after its last write is kept (%v), %d objects", err, len(objs.objs))
}
// an input of the old form goes a day after the time its key carries, with its object, never earlier
old := "20261117T090000Z." + digestOf([]byte("old"))
if err := k.vals.Put(old, base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{'o'}, 2<<20))); err != nil {
t.Fatal(err)
}
recent := now.Add(-time.Hour).Format(inputsTime) + "." + digestOf([]byte("recent"))
if err := k.vals.Put(recent, "cmVjZW50"); err != nil {
t.Fatal(err)
}
if err := k.sweep(); err != nil {
t.Fatal(err)
}
if e, _ := k.vals.KV.Get(old); e != nil || len(objs.objs) != 0 {
t.Errorf("the input of the old form is kept (%v) with %d objects", e, len(objs.objs))
}
if e, _ := k.vals.KV.Get(recent); e == nil {
t.Error("an input of the old form under a day old was deleted early")
}
// a new content the cap refuses says the fill; the same content, already kept, needs no room
full := newTestKeeperWithFill(objs, now, inputsCap-10)
if _, err := full.keep("ask-8", []byte("fits")); err != nil {
t.Fatalf("a content that fits is refused: %v", err)
}
if _, err := full.keep("ask-9", []byte("does not fit at all")); err == nil || !strings.Contains(err.Error(), "bytes kept") {
t.Errorf("the refusal does not name the fill: %v", err)
}
if _, err := full.keep("ask-10", []byte("fits")); err != nil {
t.Errorf("the content already kept is refused at the cap: %v", err)
}
}
func newTestKeeperWithFill(objs *objectsMem, now time.Time, used int64) *inputKeeper {
k := testKeeper(objs, func() time.Time { return now })
k.used = func() (int64, error) { return used, nil }
return k
}