O-2: C-S3, C-S5: stateobjects.Values over a bundle's runtime, values of any size, a replace removes the object it replaced, a deleted key leaves its object listed key-gone (milestone 47/G, issue 1966, ADR 0666 section 16)
The runtime's channel carries 16 MiB a message, so a value over the 8 MiB a put takes is split over objects (Ref.Parts). Builds: design 47 G §57, §138 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ
This commit is contained in:
@@ -0,0 +1,81 @@
|
||||
package stateobjects
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/busstate"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// ForState is a state of this module, kept through its runtime: the state by its local name, and its object store
|
||||
// `<state>-objects` (catalogue.StateObjectsName).
|
||||
func ForState(state string) Values {
|
||||
store := catalogue.StateObjectsName(state)
|
||||
return Values{KV: BundleKV{State: stdio.State(state)}, Objects: BundleObjects{Store: busstate.ObjectsOf(store)}, Store: store}
|
||||
}
|
||||
|
||||
// BundleKV is a state as a bundle reaches it.
|
||||
type BundleKV struct{ State stdio.KeptState }
|
||||
|
||||
func (b BundleKV) Get(key string) (*Entry, error) {
|
||||
e, err := b.State.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Entry{Value: e.Value, Revision: e.Revision}, nil
|
||||
}
|
||||
|
||||
func changed(err error) error {
|
||||
if errors.Is(err, stdio.ErrChanged) {
|
||||
return ErrChanged
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (b BundleKV) Create(key string, value []byte) error {
|
||||
_, err := b.State.Create(key, json.RawMessage(value))
|
||||
return changed(err)
|
||||
}
|
||||
|
||||
func (b BundleKV) Update(key string, value []byte, revision uint64) error {
|
||||
_, err := b.State.Update(key, json.RawMessage(value), revision)
|
||||
return changed(err)
|
||||
}
|
||||
|
||||
func (b BundleKV) Delete(key string) error { return b.State.Delete(key) }
|
||||
func (b BundleKV) Keys() ([]string, error) { return b.State.Keys() }
|
||||
|
||||
// BundleObjects is an object store as a bundle reaches it.
|
||||
type BundleObjects struct{ Store busstate.Objects }
|
||||
|
||||
// BundleObjectMost is the largest object a bundle puts in one call: the runtime's channel carries a message of 16 MiB, and
|
||||
// the bytes travel in it as base64.
|
||||
const BundleObjectMost = 8 << 20
|
||||
|
||||
// MostBytes is BundleObjectMost: a larger value is split over objects.
|
||||
func (BundleObjects) MostBytes() int { return BundleObjectMost }
|
||||
|
||||
func (b BundleObjects) Put(name string, data []byte) error {
|
||||
_, err := b.Store.Put(name, data, "a value kept beside its key")
|
||||
return err
|
||||
}
|
||||
|
||||
func (b BundleObjects) Get(name string) ([]byte, error) {
|
||||
d, err := b.Store.Get(name)
|
||||
return d.Data, err
|
||||
}
|
||||
|
||||
func (b BundleObjects) Delete(name string) error { return b.Store.Delete(name) }
|
||||
|
||||
func (b BundleObjects) List() ([]Object, error) {
|
||||
l, err := b.Store.List()
|
||||
out := make([]Object, 0, len(l))
|
||||
for _, o := range l {
|
||||
if !o.Deleted {
|
||||
out = append(out, Object{Name: o.Name, Size: int64(o.Size), Modified: o.Modified})
|
||||
}
|
||||
}
|
||||
return out, err
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
package stateobjects
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
)
|
||||
|
||||
// memory is a state and its object store in memory, behind the two interfaces.
|
||||
type memory struct {
|
||||
mu sync.Mutex
|
||||
rev uint64
|
||||
keys map[string]Entry
|
||||
objects map[string][]byte
|
||||
failPuts bool
|
||||
most int
|
||||
}
|
||||
|
||||
func newMemory() *memory { return &memory{keys: map[string]Entry{}, objects: map[string][]byte{}} }
|
||||
|
||||
func (m *memory) Get(key string) (*Entry, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if e, ok := m.keys[key]; ok {
|
||||
return &e, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m *memory) write(key string, value []byte, revision *uint64) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
e, has := m.keys[key]
|
||||
if (revision == nil && has) || (revision != nil && (!has || e.Revision != *revision)) {
|
||||
return ErrChanged
|
||||
}
|
||||
m.rev++
|
||||
m.keys[key] = Entry{Value: value, Revision: m.rev}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *memory) Create(key string, value []byte) error { return m.write(key, value, nil) }
|
||||
func (m *memory) Update(key string, value []byte, revision uint64) error {
|
||||
return m.write(key, value, &revision)
|
||||
}
|
||||
|
||||
func (m *memory) Delete(key string) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
delete(m.keys, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *memory) Keys() ([]string, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
var out []string
|
||||
for k := range m.keys {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (m *memory) Put(name string, data []byte) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.failPuts {
|
||||
return errors.New("the store is full")
|
||||
}
|
||||
m.objects[name] = data
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *memory) GetObject(name string) ([]byte, error) { return m.Object(name) }
|
||||
|
||||
func (m *memory) values() Values { return Values{KV: m, Objects: memObjects{m}, Store: "s-objects"} }
|
||||
|
||||
// Object is an object's bytes.
|
||||
func (m *memory) Object(name string) ([]byte, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
d, ok := m.objects[name]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("no object %s", name)
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// memObjects is the memory's object store side (Put is the memory's own).
|
||||
type memObjects struct{ m *memory }
|
||||
|
||||
func (o memObjects) MostBytes() int { return o.m.most }
|
||||
func (o memObjects) Put(name string, data []byte) error { return o.m.Put(name, data) }
|
||||
func (o memObjects) Get(name string) ([]byte, error) { return o.m.Object(name) }
|
||||
func (o memObjects) Delete(name string) error {
|
||||
o.m.mu.Lock()
|
||||
defer o.m.mu.Unlock()
|
||||
delete(o.m.objects, name)
|
||||
return nil
|
||||
}
|
||||
func (o memObjects) List() ([]Object, error) {
|
||||
o.m.mu.Lock()
|
||||
defer o.m.mu.Unlock()
|
||||
var out []Object
|
||||
for n, d := range o.m.objects {
|
||||
out = append(out, Object{Name: n, Size: int64(len(d))})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// runtimeOf serves one bundle tool, "run", against the memory as the tool runner would answer the state's and the object
|
||||
// store's calls, and returns what the tool answered. The tool runs run in the bundle, which reaches the memory through
|
||||
// stdio.State and busstate.Objects.
|
||||
func runtimeOf(t *testing.T, m *memory, run func() (any, error)) any {
|
||||
t.Helper()
|
||||
t.Setenv(stdio.ServedAs, "owner")
|
||||
inR, inW := io.Pipe()
|
||||
outR, outW := io.Pipe()
|
||||
go func() {
|
||||
_ = stdio.ServeOn("", []stdio.Tool{{Name: "run", Run: func(map[string]any) (any, error) { return run() }}}, inR, outW)
|
||||
}()
|
||||
var wmu sync.Mutex
|
||||
send := func(v map[string]any) {
|
||||
v["jsonrpc"] = "2.0"
|
||||
b, _ := json.Marshal(v)
|
||||
wmu.Lock()
|
||||
defer wmu.Unlock()
|
||||
_, _ = inW.Write(append(b, '\n'))
|
||||
}
|
||||
answered := make(chan any, 1)
|
||||
go func() {
|
||||
lines := bufio.NewScanner(outR)
|
||||
lines.Buffer(make([]byte, 1<<20), 1<<28)
|
||||
for lines.Scan() {
|
||||
var in struct {
|
||||
ID json.RawMessage `json:"id"`
|
||||
Method string `json:"method"`
|
||||
Params json.RawMessage `json:"params"`
|
||||
Result json.RawMessage `json:"result"`
|
||||
}
|
||||
if json.Unmarshal(lines.Bytes(), &in) != nil {
|
||||
continue
|
||||
}
|
||||
if !strings.HasPrefix(in.Method, "mesh/") {
|
||||
if string(in.ID) == "1" {
|
||||
answered <- string(in.Result)
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
go func() {
|
||||
res, err := m.serve(in.Method, in.Params)
|
||||
a := map[string]any{"id": json.RawMessage(in.ID)}
|
||||
if err != nil {
|
||||
code := -32000
|
||||
if errors.Is(err, ErrChanged) {
|
||||
code = stdio.CodeChanged
|
||||
}
|
||||
a["error"] = map[string]any{"code": code, "message": err.Error()}
|
||||
} else {
|
||||
a["result"] = res
|
||||
}
|
||||
send(a)
|
||||
}()
|
||||
}
|
||||
}()
|
||||
send(map[string]any{"id": 1, "method": "tools/call", "params": map[string]any{"name": "run"}})
|
||||
select {
|
||||
case a := <-answered:
|
||||
return a
|
||||
case <-time.After(20 * time.Second):
|
||||
t.Fatal("the bundle said nothing")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (m *memory) serve(method string, raw json.RawMessage) (any, error) {
|
||||
var p struct {
|
||||
Key string `json:"key"`
|
||||
Value json.RawMessage `json:"value"`
|
||||
Revision uint64 `json:"revision"`
|
||||
Name string `json:"name"`
|
||||
Data []byte `json:"data"`
|
||||
}
|
||||
_ = json.Unmarshal(raw, &p)
|
||||
switch method {
|
||||
case "mesh/state.get":
|
||||
e, _ := m.Get(p.Key)
|
||||
if e == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return map[string]any{"key": p.Key, "value": json.RawMessage(e.Value), "revision": e.Revision}, nil
|
||||
case "mesh/state.create":
|
||||
return map[string]any{"revision": 1}, m.Create(p.Key, p.Value)
|
||||
case "mesh/state.update":
|
||||
return map[string]any{"revision": 1}, m.Update(p.Key, p.Value, p.Revision)
|
||||
case "mesh/state.delete":
|
||||
return buscalls.Done{}, m.Delete(p.Key)
|
||||
case "mesh/state.keys":
|
||||
return m.Keys()
|
||||
case buscalls.ObjectsPut:
|
||||
return buscalls.ObjectInfo{Name: p.Name, Size: uint64(len(p.Data))}, m.Put(p.Name, p.Data)
|
||||
case buscalls.ObjectsGet:
|
||||
d, err := m.Object(p.Name)
|
||||
return buscalls.ObjectData{Info: buscalls.ObjectInfo{Name: p.Name}, Data: d}, err
|
||||
case buscalls.ObjectsDelete:
|
||||
return buscalls.Done{}, memObjects{m}.Delete(p.Name)
|
||||
case buscalls.ObjectsList:
|
||||
l, _ := memObjects{m}.List()
|
||||
var out []buscalls.ObjectInfo
|
||||
for _, o := range l {
|
||||
out = append(out, buscalls.ObjectInfo{Name: o.Name, Size: uint64(o.Size)})
|
||||
}
|
||||
return buscalls.ObjectsListed{Objects: out}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("no %s", method)
|
||||
}
|
||||
@@ -27,6 +27,9 @@ type Ref struct {
|
||||
Object string `json:"object"`
|
||||
// Size is the object's size in bytes.
|
||||
Size int64 `json:"size"`
|
||||
// Parts is how many objects the bytes are split over, named `<Object>.<n>` from 0, when the store takes no object as
|
||||
// large as the value (a bundle's runtime carries a message of 16 MiB); zero is one object named Object.
|
||||
Parts int `json:"parts,omitempty"`
|
||||
// Digest is the hex SHA-256 of the object's bytes; a reader checks it after the read.
|
||||
Digest string `json:"sha256"`
|
||||
// Form is FormVersion.
|
||||
@@ -34,7 +37,7 @@ type Ref struct {
|
||||
}
|
||||
|
||||
func (r Ref) valid() bool {
|
||||
if r.Store == "" || r.Object == "" || r.Size < 0 || r.Form != FormVersion || len(r.Digest) != 64 {
|
||||
if r.Store == "" || r.Object == "" || r.Size < 0 || r.Parts < 0 || r.Form != FormVersion || len(r.Digest) != 64 {
|
||||
return false
|
||||
}
|
||||
_, err := hex.DecodeString(r.Digest)
|
||||
|
||||
@@ -0,0 +1,303 @@
|
||||
package stateobjects
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/kvlimit"
|
||||
)
|
||||
|
||||
// ErrChanged is a write refused because the key was written by another writer first; nothing was written.
|
||||
var ErrChanged = errors.New("the key was written by another writer first")
|
||||
|
||||
// Entry is a key's current value and its revision.
|
||||
type Entry struct {
|
||||
Value []byte
|
||||
Revision uint64
|
||||
}
|
||||
|
||||
// KV is the state's key-value side, as a bundle reaches it through its runtime or a process through its own JetStream.
|
||||
type KV interface {
|
||||
// Get is the key's current entry, or nil when it has none.
|
||||
Get(key string) (*Entry, error)
|
||||
// Create writes the key only when it has none, Update only while it is at revision; either refuses as ErrChanged.
|
||||
Create(key string, value []byte) error
|
||||
Update(key string, value []byte, revision uint64) error
|
||||
Delete(key string) error
|
||||
// Keys is every key with a value.
|
||||
Keys() ([]string, error)
|
||||
}
|
||||
|
||||
// Object is one object an object store holds.
|
||||
type Object struct {
|
||||
Name string
|
||||
Size int64
|
||||
Modified time.Time
|
||||
}
|
||||
|
||||
// Objects is the object store beside the state.
|
||||
type Objects interface {
|
||||
// MostBytes is the largest object it takes in one put, or zero for no limit: a value over it is split over objects.
|
||||
MostBytes() int
|
||||
Put(name string, data []byte) error
|
||||
Get(name string) ([]byte, error)
|
||||
Delete(name string) error
|
||||
List() ([]Object, error)
|
||||
}
|
||||
|
||||
// Values is a state whose values may be of any size: a value that fits the bucket is kept in its key, a larger one in
|
||||
// the object store beside it with its reference in the key (ADR 0666 §16). Store is the object store's name as a reader
|
||||
// asks for it, written into every reference.
|
||||
type Values struct {
|
||||
KV KV
|
||||
Objects Objects
|
||||
Store string
|
||||
// Limit is the most a key's value may hold, headers included; zero is kvlimit.ModuleStateValueBytes.
|
||||
Limit int
|
||||
}
|
||||
|
||||
func (v Values) inlineMost() int {
|
||||
limit := v.Limit
|
||||
if limit <= 0 {
|
||||
limit = kvlimit.ModuleStateValueBytes
|
||||
}
|
||||
return limit - kvlimit.HeaderRoom
|
||||
}
|
||||
|
||||
func digestOf(data []byte) string {
|
||||
sum := sha256.Sum256(data)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// objectName is a large value's object, `<key>.<digest>`: the same value is the same object, a changed one another.
|
||||
func objectName(key, digest string) string { return key + "." + digest }
|
||||
|
||||
// Put writes the value as JSON under key: inline up to the limit, else as an object `<key>.<sha256>` with its reference
|
||||
// in the key. A replace removes the object it replaced in the same act; a write another writer won first is ErrChanged
|
||||
// and leaves what it found. A value that is itself exactly a reference is refused.
|
||||
func (v Values) Put(key string, value any) error {
|
||||
data, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the value for %s: %w", key, err)
|
||||
}
|
||||
if _, isRef := ReadRef(data); isRef {
|
||||
return fmt.Errorf("%s: %w", key, errLooksLikeRef)
|
||||
}
|
||||
old, err := v.KV.Get(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var oldRef Ref
|
||||
var hadRef bool
|
||||
if old != nil {
|
||||
oldRef, hadRef = ReadRef(old.Value)
|
||||
}
|
||||
body, object := data, ""
|
||||
var newRef Ref
|
||||
if len(data) > v.inlineMost() {
|
||||
ref := Ref{Store: v.Store, Object: objectName(key, digestOf(data)), Size: int64(len(data)), Digest: digestOf(data), Form: FormVersion}
|
||||
object = ref.Object
|
||||
if !(hadRef && oldRef.Object == object) {
|
||||
parts, err := putObject(v.Objects, object, data)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the object for %s: %w", key, err)
|
||||
}
|
||||
ref.Parts = parts
|
||||
} else {
|
||||
ref.Parts = oldRef.Parts
|
||||
}
|
||||
body = Wrap(ref)
|
||||
newRef = ref
|
||||
}
|
||||
if old == nil {
|
||||
err = v.KV.Create(key, body)
|
||||
} else {
|
||||
err = v.KV.Update(key, body, old.Revision)
|
||||
}
|
||||
if err != nil {
|
||||
if object != "" && !(hadRef && oldRef.Object == object) {
|
||||
_ = deleteObject(v.Objects, newRef)
|
||||
}
|
||||
return err
|
||||
}
|
||||
if hadRef && oldRef.Object != object {
|
||||
if err := deleteObject(v.Objects, oldRef); err != nil {
|
||||
return fmt.Errorf("%s is written; the object it replaced, %s, could not be removed: %w", key, oldRef.Object, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get reads the key's value into into and says whether the key has one. A value kept as an object is fetched and
|
||||
// checked against its reference's size and digest.
|
||||
func (v Values) Get(key string, into any) (bool, error) {
|
||||
e, err := v.KV.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return false, err
|
||||
}
|
||||
data := e.Value
|
||||
if ref, ok := ReadRef(e.Value); ok {
|
||||
if data, err = getObject(v.Objects, ref); err != nil {
|
||||
return false, fmt.Errorf("the object %s of %s: %w", ref.Object, key, err)
|
||||
}
|
||||
if int64(len(data)) != ref.Size || digestOf(data) != ref.Digest {
|
||||
return false, fmt.Errorf("the object %s of %s is not the one its reference names (size or digest differ)", ref.Object, key)
|
||||
}
|
||||
}
|
||||
return true, json.Unmarshal(data, into)
|
||||
}
|
||||
|
||||
// Delete removes the key. Its object stays, listed as key-gone, until the drop tool removes it on a person's word
|
||||
// (ADR 0666 §16, ADR 0595 §4).
|
||||
func (v Values) Delete(key string) error { return v.KV.Delete(key) }
|
||||
|
||||
// Kept is one object of a state's store as the reading tool shows it.
|
||||
type Kept struct {
|
||||
Name string `json:"name"`
|
||||
Size int64 `json:"size"`
|
||||
Modified time.Time `json:"modified,omitzero"`
|
||||
// Key is the key whose value names it; for an object whose key is gone, the key it was written for.
|
||||
Key string `json:"key"`
|
||||
// KeyGone says no key names the object any longer.
|
||||
KeyGone bool `json:"key-gone,omitempty"`
|
||||
}
|
||||
|
||||
// Kept lists every object of the store with the key naming it, or key-gone, by name.
|
||||
func (v Values) Kept() ([]Kept, error) {
|
||||
live, err := v.live()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
objects, err := v.Objects.List()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]Kept, 0, len(objects))
|
||||
for _, o := range objects {
|
||||
k := Kept{Name: o.Name, Size: o.Size, Modified: o.Modified}
|
||||
if key, ok := live[o.Name]; ok {
|
||||
k.Key = key
|
||||
} else {
|
||||
k.KeyGone = true
|
||||
k.Key = keyOf(o.Name)
|
||||
}
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// live is the objects some key names now, by object name to key.
|
||||
func (v Values) live() (map[string]string, error) {
|
||||
keys, err := v.KV.Keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
live := map[string]string{}
|
||||
for _, key := range keys {
|
||||
e, err := v.KV.Get(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if e == nil {
|
||||
continue
|
||||
}
|
||||
if ref, ok := ReadRef(e.Value); ok {
|
||||
for _, name := range ref.names() {
|
||||
live[name] = key
|
||||
}
|
||||
}
|
||||
}
|
||||
return live, nil
|
||||
}
|
||||
|
||||
// DropKeyGone removes the objects whose key is gone, and only those, answering their names. It is the act of the drop
|
||||
// tool; a live object, one a key names, is never removed.
|
||||
func (v Values) DropKeyGone() ([]string, error) {
|
||||
kept, err := v.Kept()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var dropped []string
|
||||
for _, k := range kept {
|
||||
if !k.KeyGone {
|
||||
continue
|
||||
}
|
||||
if err := v.Objects.Delete(k.Name); err != nil {
|
||||
return dropped, fmt.Errorf("the object %s: %w", k.Name, err)
|
||||
}
|
||||
dropped = append(dropped, k.Name)
|
||||
}
|
||||
return dropped, nil
|
||||
}
|
||||
|
||||
// names are the objects a reference stands for.
|
||||
func (r Ref) names() []string {
|
||||
if r.Parts <= 1 {
|
||||
return []string{r.Object}
|
||||
}
|
||||
out := make([]string, r.Parts)
|
||||
for i := range out {
|
||||
out[i] = fmt.Sprintf("%s.%d", r.Object, i)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// keyOf is the key an object `<key>.<digest>` or `<key>.<digest>.<n>` was written for.
|
||||
func keyOf(object string) string {
|
||||
isDigest := func(s string) bool { _, err := hex.DecodeString(s); return len(s) == 64 && err == nil }
|
||||
parts := strings.Split(object, ".")
|
||||
if n := len(parts); n >= 3 && isDigest(parts[n-2]) {
|
||||
return strings.Join(parts[:n-2], ".")
|
||||
}
|
||||
if n := len(parts); n >= 2 && isDigest(parts[n-1]) {
|
||||
return strings.Join(parts[:n-1], ".")
|
||||
}
|
||||
return object
|
||||
}
|
||||
|
||||
// putObject stores data as one object, or over `<name>.<n>` when the store takes none so large; it answers the parts
|
||||
// (zero for one object) and removes what it stored when a part fails.
|
||||
func putObject(o Objects, name string, data []byte) (int, error) {
|
||||
most := o.MostBytes()
|
||||
if most <= 0 || len(data) <= most {
|
||||
return 0, o.Put(name, data)
|
||||
}
|
||||
ref := Ref{Object: name, Parts: (len(data) + most - 1) / most}
|
||||
for i, part := range ref.names() {
|
||||
from, to := i*most, min((i+1)*most, len(data))
|
||||
if err := o.Put(part, data[from:to]); err != nil {
|
||||
_ = deleteObject(o, Ref{Object: name, Parts: i})
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
return ref.Parts, nil
|
||||
}
|
||||
|
||||
func getObject(o Objects, ref Ref) ([]byte, error) {
|
||||
var out []byte
|
||||
for _, name := range ref.names() {
|
||||
d, err := o.Get(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, d...)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func deleteObject(o Objects, ref Ref) error {
|
||||
var first error
|
||||
for _, name := range ref.names() {
|
||||
if err := o.Delete(name); err != nil && first == nil {
|
||||
first = err
|
||||
}
|
||||
}
|
||||
return first
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
package stateobjects
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/kvlimit"
|
||||
)
|
||||
|
||||
// big is a JSON value of about n bytes.
|
||||
func big(n int) map[string]string { return map[string]string{"text": strings.Repeat("x", n)} }
|
||||
|
||||
// valueRows is what every backend of Values holds: a value of 1x, 10x and 100x the limit is read back whole, a replace
|
||||
// leaves one object, a delete leaves one listed key-gone (ADR 0666 §16, C-Q1).
|
||||
func valueRows(t *testing.T, v Values) {
|
||||
t.Helper()
|
||||
limit := kvlimit.ModuleStateValueBytes
|
||||
for _, times := range []int{1, 10, 100} {
|
||||
key := map[int]string{1: "one", 10: "ten", 100: "hundred"}[times]
|
||||
want := big(times * limit)
|
||||
if err := v.Put(key, want); err != nil {
|
||||
t.Errorf("%dx: put: %v", times, err)
|
||||
return
|
||||
}
|
||||
var got map[string]string
|
||||
if ok, err := v.Get(key, &got); err != nil || !ok || got["text"] != want["text"] {
|
||||
t.Errorf("%dx: read back %v, found %v, %v", times, len(got["text"]), ok, err)
|
||||
return
|
||||
}
|
||||
e, _ := v.KV.Get(key)
|
||||
if _, isRef := ReadRef(e.Value); !isRef {
|
||||
t.Errorf("%dx: the key holds the value, not a reference", times)
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := v.Put("small", map[string]int{"a": 1}); err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
if e, _ := v.KV.Get("small"); !bytes.Equal(e.Value, []byte(`{"a":1}`)) {
|
||||
t.Errorf("a small value is kept in its key, got %s", e.Value)
|
||||
return
|
||||
}
|
||||
// a replace removes the object it replaced; the same value again leaves its object
|
||||
before, _ := v.Objects.List()
|
||||
if err := v.Put("one", big(2*limit)); err != nil { // 10x key
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
if err := v.Put("one", big(2*limit)); err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
after, _ := v.Objects.List()
|
||||
if len(after) != len(before) {
|
||||
t.Errorf("a replace left %d objects, %d before it", len(after), len(before))
|
||||
return
|
||||
}
|
||||
// a replace by a small value removes the object too
|
||||
if err := v.Put("one", "small now"); err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
if now, _ := v.Objects.List(); len(now) != len(before)-1 {
|
||||
t.Errorf("replacing a large value by a small one left %d objects, want %d", len(now), len(before)-1)
|
||||
return
|
||||
}
|
||||
// a delete leaves the object, listed key-gone, and the drop removes it alone
|
||||
if err := v.Delete("ten"); err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
kept, err := v.Kept()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
var gone []string
|
||||
for _, k := range kept {
|
||||
if k.KeyGone {
|
||||
gone = append(gone, k.Key)
|
||||
}
|
||||
}
|
||||
if len(gone) != 1 || gone[0] != "ten" {
|
||||
t.Errorf("the objects whose key is gone: %v, want ten's", gone)
|
||||
return
|
||||
}
|
||||
dropped, err := v.DropKeyGone()
|
||||
if err != nil || len(dropped) != 1 {
|
||||
t.Errorf("dropped %v, %v", dropped, err)
|
||||
return
|
||||
}
|
||||
if left, _ := v.Objects.List(); len(left) != len(kept)-1 {
|
||||
t.Errorf("the drop left %d objects of %d", len(left), len(kept))
|
||||
return
|
||||
}
|
||||
if err := v.Put("ref", json.RawMessage(Wrap(Ref{Store: "s", Object: "o", Digest: strings.Repeat("a", 64), Form: FormVersion}))); err == nil {
|
||||
t.Error("a value that is exactly a reference was accepted")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0666 §16, C-S3, C-S5: values of 1x, 10x and 100x the limit through a bundle's runtime, whole.
|
||||
func TestValuesOfAnySizeThroughTheRuntime(t *testing.T) {
|
||||
m := newMemory()
|
||||
out := runtimeOf(t, m, func() (any, error) {
|
||||
v := ForState("views")
|
||||
valueRows(t, v)
|
||||
return "done", nil
|
||||
})
|
||||
if !strings.Contains(out.(string), "done") {
|
||||
t.Fatalf("the bundle answered %v", out)
|
||||
}
|
||||
}
|
||||
|
||||
// A write another writer won first leaves the object it put removed and the key as it found it.
|
||||
func TestAWriteLostLeavesNoObjectBehind(t *testing.T) {
|
||||
m := newMemory()
|
||||
v := m.values()
|
||||
if err := v.Put("k", big(3*kvlimit.ModuleStateValueBytes)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// another writer changes the key between this writer's read and write
|
||||
racing := &raceKV{KV: m, then: func() { _ = m.Update("k", []byte(`"other"`), 1) }}
|
||||
lost := Values{KV: racing, Objects: memObjects{m}, Store: "s"}
|
||||
if err := lost.Put("k", big(4*kvlimit.ModuleStateValueBytes)); err == nil || err != ErrChanged {
|
||||
t.Fatalf("the lost write: %v", err)
|
||||
}
|
||||
if l, _ := (memObjects{m}).List(); len(l) != 1 {
|
||||
t.Fatalf("%d objects, want the first value's only", len(l))
|
||||
}
|
||||
m.failPuts = true
|
||||
if err := v.Put("k2", big(3*kvlimit.ModuleStateValueBytes)); err == nil {
|
||||
t.Fatal("a put into a full store was accepted")
|
||||
}
|
||||
if e, _ := m.Get("k2"); e != nil {
|
||||
t.Fatal("the key was written though its object was not")
|
||||
}
|
||||
}
|
||||
|
||||
// raceKV runs then once, after the first Get, as another writer would.
|
||||
type raceKV struct {
|
||||
KV
|
||||
then func()
|
||||
}
|
||||
|
||||
func (r *raceKV) Get(key string) (*Entry, error) {
|
||||
e, err := r.KV.Get(key)
|
||||
if r.then != nil {
|
||||
then := r.then
|
||||
r.then = nil
|
||||
then()
|
||||
}
|
||||
return e, err
|
||||
}
|
||||
|
||||
// A store that takes no object as large as the value gets it split over objects, read back whole, replaced and dropped as
|
||||
// one (a bundle's runtime carries a message of 16 MiB).
|
||||
func TestAValueLargerThanTheStoreTakesIsSplitOverObjects(t *testing.T) {
|
||||
m := newMemory()
|
||||
m.most = 100_000
|
||||
v := m.values()
|
||||
want := big(5*kvlimit.ModuleStateValueBytes + 10)
|
||||
if err := v.Put("k", want); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if l, _ := (memObjects{m}).List(); len(l) < 13 {
|
||||
t.Fatalf("%d objects, want the value split over at least 13", len(l))
|
||||
}
|
||||
var got map[string]string
|
||||
if ok, err := v.Get("k", &got); err != nil || !ok || got["text"] != want["text"] {
|
||||
t.Fatalf("read back %d, %v, %v", len(got["text"]), ok, err)
|
||||
}
|
||||
if err := v.Delete("k"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kept, _ := v.Kept()
|
||||
for _, k := range kept {
|
||||
if !k.KeyGone || k.Key != "k" {
|
||||
t.Fatalf("a part %+v is not listed as key-gone for k", k)
|
||||
}
|
||||
}
|
||||
if dropped, err := v.DropKeyGone(); err != nil || len(dropped) != len(kept) {
|
||||
t.Fatalf("dropped %v of %d, %v", dropped, len(kept), err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user