diff --git a/go/stateobjects/bundle.go b/go/stateobjects/bundle.go index 930e554b..d8ada3af 100644 --- a/go/stateobjects/bundle.go +++ b/go/stateobjects/bundle.go @@ -54,7 +54,7 @@ type BundleObjects struct{ Store busstate.Objects } // the bytes travel in it as base64. const BundleObjectMost = 8 << 20 -// MostBytes is BundleObjectMost: a larger value is split over objects. +// MostBytes is BundleObjectMost: a larger value is refused, loudly, until the design decides how a bundle keeps one. func (BundleObjects) MostBytes() int { return BundleObjectMost } func (b BundleObjects) Put(name string, data []byte) error { diff --git a/go/stateobjects/ref.go b/go/stateobjects/ref.go index 008f4e55..1c76fc79 100644 --- a/go/stateobjects/ref.go +++ b/go/stateobjects/ref.go @@ -27,9 +27,6 @@ type Ref struct { Object string `json:"object"` // Size is the object's size in bytes. Size int64 `json:"size"` - // Parts is how many objects the bytes are split over, named `.` from 0, when the store takes no object as - // large as the value (a bundle's runtime carries a message of 16 MiB); zero is one object named Object. - Parts int `json:"parts,omitempty"` // Digest is the hex SHA-256 of the object's bytes; a reader checks it after the read. Digest string `json:"sha256"` // Form is FormVersion. @@ -37,7 +34,7 @@ type Ref struct { } func (r Ref) valid() bool { - if r.Store == "" || r.Object == "" || r.Size < 0 || r.Parts < 0 || r.Form != FormVersion || len(r.Digest) != 64 { + if r.Store == "" || r.Object == "" || r.Size < 0 || r.Form != FormVersion || len(r.Digest) != 64 { return false } _, err := hex.DecodeString(r.Digest) diff --git a/go/stateobjects/send.go b/go/stateobjects/send.go index 361588b1..2f6d98e7 100644 --- a/go/stateobjects/send.go +++ b/go/stateobjects/send.go @@ -109,7 +109,7 @@ func (s Sender) Send(label string, body []byte) ([]byte, error) { return nil, fmt.Errorf("%s: a bundle sends at most %d bytes of body, the runtime's channel carrying a message of 16 MiB; this one is %d", label, most, len(body)) } ref := Ref{Store: s.Store, Object: objectName(label, digestOf(body)), Size: int64(len(body)), Digest: digestOf(body), Form: FormVersion} - if _, err := putObject(s.Objects, ref.Object, body); err != nil { + if err := s.Objects.Put(ref.Object, body); err != nil { return nil, fmt.Errorf("the object for %s: %w", label, err) } if s.Log != nil { @@ -143,9 +143,6 @@ func Receive(body []byte, fetch Fetch) ([]byte, error) { // BundleFetch reads a sent object through the runtime, which serves it under the standing grant the grants part derives // from the manifests (buscalls.ObjectsSentGet). func BundleFetch(ref Ref) ([]byte, error) { - if ref.Parts > 1 { - return nil, fmt.Errorf("the object %s is split over %d objects, which a bundle does not read: a sender in a bundle never splits one", ref.Object, ref.Parts) - } raw, err := stdio.BusState(buscalls.ObjectsSentGet, buscalls.ObjectsSentGetRequest{Store: ref.Store, Object: ref.Object, Size: ref.Size, Digest: ref.Digest}) if err != nil { return nil, err @@ -159,5 +156,5 @@ func BundleFetch(ref Ref) ([]byte, error) { // ProcessFetch reads a sent object through the reader's own object store. func ProcessFetch(o Objects) Fetch { - return func(ref Ref) ([]byte, error) { return getObject(o, ref) } + return func(ref Ref) ([]byte, error) { return o.Get(ref.Object) } } diff --git a/go/stateobjects/values.go b/go/stateobjects/values.go index 4c36cb90..b9d52847 100644 --- a/go/stateobjects/values.go +++ b/go/stateobjects/values.go @@ -43,7 +43,7 @@ type Object struct { // Objects is the object store beside the state. type Objects interface { - // MostBytes is the largest object it takes in one put, or zero for no limit: a value over it is split over objects. + // MostBytes is the largest object it takes in one put, or zero for no limit: a value over it is refused, loudly. MostBytes() int Put(name string, data []byte) error Get(name string) ([]byte, error) @@ -79,8 +79,9 @@ func digestOf(data []byte) string { func objectName(key, digest string) string { return key + "." + digest } // Put writes the value as JSON under key: inline up to the limit, else as an object `.` with its reference -// in the key. A replace removes the object it replaced in the same act; a write another writer won first is ErrChanged -// and leaves what it found. A value that is itself exactly a reference is refused. +// in the key. A replace leaves the object it replaced, as a past value may still name it (ADR 0595 §4): it is listed +// key-gone until the drop tool removes it on a person's word. A write another writer won first is ErrChanged and leaves +// what it found. A value that is itself exactly a reference is refused. func (v Values) Put(key string, value any) error { data, err := json.Marshal(value) if err != nil { @@ -99,21 +100,18 @@ func (v Values) Put(key string, value any) error { oldRef, hadRef = ReadRef(old.Value) } body, object := data, "" - var newRef Ref if len(data) > v.inlineMost() { ref := Ref{Store: v.Store, Object: objectName(key, digestOf(data)), Size: int64(len(data)), Digest: digestOf(data), Form: FormVersion} object = ref.Object + if most := v.Objects.MostBytes(); most > 0 && len(data) > most { + return fmt.Errorf("the value for %s is %d bytes, over the %d an object may hold through this runtime: it is refused", key, len(data), most) + } if !(hadRef && oldRef.Object == object) { - parts, err := putObject(v.Objects, object, data) - if err != nil { + if err := v.Objects.Put(object, data); err != nil { return fmt.Errorf("the object for %s: %w", key, err) } - ref.Parts = parts - } else { - ref.Parts = oldRef.Parts } body = Wrap(ref) - newRef = ref } if old == nil { err = v.KV.Create(key, body) @@ -122,15 +120,11 @@ func (v Values) Put(key string, value any) error { } if err != nil { if object != "" && !(hadRef && oldRef.Object == object) { - _ = deleteObject(v.Objects, newRef) + // an object no key and no past value names: made for this write alone + _ = v.Objects.Delete(object) } return err } - if hadRef && oldRef.Object != object { - if err := deleteObject(v.Objects, oldRef); err != nil { - return fmt.Errorf("%s is written; the object it replaced, %s, could not be removed: %w", key, oldRef.Object, err) - } - } return nil } @@ -143,7 +137,7 @@ func (v Values) Get(key string, into any) (bool, error) { } data := e.Value if ref, ok := ReadRef(e.Value); ok { - if data, err = getObject(v.Objects, ref); err != nil { + if data, err = v.Objects.Get(ref.Object); err != nil { return false, fmt.Errorf("the object %s of %s: %w", ref.Object, key, err) } if int64(len(data)) != ref.Size || digestOf(data) != ref.Digest { @@ -209,9 +203,7 @@ func (v Values) live() (map[string]string, error) { continue } if ref, ok := ReadRef(e.Value); ok { - for _, name := range ref.names() { - live[name] = key - } + live[ref.Object] = key } } return live, nil @@ -237,67 +229,12 @@ func (v Values) DropKeyGone() ([]string, error) { return dropped, nil } -// names are the objects a reference stands for. -func (r Ref) names() []string { - if r.Parts <= 1 { - return []string{r.Object} - } - out := make([]string, r.Parts) - for i := range out { - out[i] = fmt.Sprintf("%s.%d", r.Object, i) - } - return out -} - -// keyOf is the key an object `.` or `..` was written for. +// keyOf is the key an object `.` was written for. func keyOf(object string) string { - isDigest := func(s string) bool { _, err := hex.DecodeString(s); return len(s) == 64 && err == nil } - parts := strings.Split(object, ".") - if n := len(parts); n >= 3 && isDigest(parts[n-2]) { - return strings.Join(parts[:n-2], ".") - } - if n := len(parts); n >= 2 && isDigest(parts[n-1]) { - return strings.Join(parts[:n-1], ".") + if i := strings.LastIndex(object, "."); i > 0 { + if _, err := hex.DecodeString(object[i+1:]); err == nil && len(object)-i-1 == 64 { + return object[:i] + } } return object } - -// putObject stores data as one object, or over `.` when the store takes none so large; it answers the parts -// (zero for one object) and removes what it stored when a part fails. -func putObject(o Objects, name string, data []byte) (int, error) { - most := o.MostBytes() - if most <= 0 || len(data) <= most { - return 0, o.Put(name, data) - } - ref := Ref{Object: name, Parts: (len(data) + most - 1) / most} - for i, part := range ref.names() { - from, to := i*most, min((i+1)*most, len(data)) - if err := o.Put(part, data[from:to]); err != nil { - _ = deleteObject(o, Ref{Object: name, Parts: i}) - return 0, err - } - } - return ref.Parts, nil -} - -func getObject(o Objects, ref Ref) ([]byte, error) { - var out []byte - for _, name := range ref.names() { - d, err := o.Get(name) - if err != nil { - return nil, err - } - out = append(out, d...) - } - return out, nil -} - -func deleteObject(o Objects, ref Ref) error { - var first error - for _, name := range ref.names() { - if err := o.Delete(name); err != nil && first == nil { - first = err - } - } - return first -} diff --git a/go/stateobjects/values_integration_test.go b/go/stateobjects/values_integration_test.go index b744836b..ab86aa0b 100644 --- a/go/stateobjects/values_integration_test.go +++ b/go/stateobjects/values_integration_test.go @@ -56,7 +56,7 @@ func TestStateObjectsWholeAgainstTheBus(t *testing.T) { } } }) - t.Run("values of 1x, 10x and 100x the limit", func(t *testing.T) { valueRows(t, OverBus(kv, store, "m_views-objects")) }) + t.Run("values of 1x, 10x and 100x the limit", func(t *testing.T) { valueRows(t, OverBus(kv, store, "m_views-objects"), 1, 10, 100) }) t.Run("the tools read and drop what a deleted key left", func(t *testing.T) { v := OverBus(kv, store, "m_views-objects") if err := v.Put("tool-key", big(2*256*1024)); err != nil { diff --git a/go/stateobjects/values_test.go b/go/stateobjects/values_test.go index 16f93329..06d14965 100644 --- a/go/stateobjects/values_test.go +++ b/go/stateobjects/values_test.go @@ -3,6 +3,7 @@ package stateobjects import ( "bytes" "encoding/json" + "slices" "strings" "testing" @@ -13,11 +14,11 @@ import ( func big(n int) map[string]string { return map[string]string{"text": strings.Repeat("x", n)} } // valueRows is what every backend of Values holds: a value of 1x, 10x and 100x the limit is read back whole, a replace -// leaves one object, a delete leaves one listed key-gone (ADR 0666 §16, C-Q1). -func valueRows(t *testing.T, v Values) { +// leaves the old object and a delete leaves its object, both listed key-gone until the drop (ADR 0666 §16, ADR 0595 §4). +func valueRows(t *testing.T, v Values, multiples ...int) { t.Helper() limit := kvlimit.ModuleStateValueBytes - for _, times := range []int{1, 10, 100} { + for _, times := range multiples { key := map[int]string{1: "one", 10: "ten", 100: "hundred"}[times] want := big(times * limit) if err := v.Put(key, want); err != nil { @@ -43,9 +44,9 @@ func valueRows(t *testing.T, v Values) { t.Errorf("a small value is kept in its key, got %s", e.Value) return } - // a replace removes the object it replaced; the same value again leaves its object + // a replace leaves the object it replaced, listed key-gone (ADR 0595 §4); the same value again adds none before, _ := v.Objects.List() - if err := v.Put("one", big(2*limit)); err != nil { // 10x key + if err := v.Put("one", big(2*limit)); err != nil { t.Error(err) return } @@ -54,17 +55,16 @@ func valueRows(t *testing.T, v Values) { return } after, _ := v.Objects.List() - if len(after) != len(before) { - t.Errorf("a replace left %d objects, %d before it", len(after), len(before)) + if len(after) != len(before)+1 { + t.Errorf("a replace left %d objects, %d before it: the old one must stay", len(after), len(before)) return } - // a replace by a small value removes the object too if err := v.Put("one", "small now"); err != nil { t.Error(err) return } - if now, _ := v.Objects.List(); len(now) != len(before)-1 { - t.Errorf("replacing a large value by a small one left %d objects, want %d", len(now), len(before)-1) + if now, _ := v.Objects.List(); len(now) != len(after) { + t.Errorf("replacing by a small value changed the objects: %d, want %d", len(now), len(after)) return } // a delete leaves the object, listed key-gone, and the drop removes it alone @@ -83,16 +83,17 @@ func valueRows(t *testing.T, v Values) { gone = append(gone, k.Key) } } - if len(gone) != 1 || gone[0] != "ten" { - t.Errorf("the objects whose key is gone: %v, want ten's", gone) + slices.Sort(gone) + if !slices.Equal(gone, []string{"one", "one", "ten"}) { + t.Errorf("the objects whose key is gone: %v, want the two replaced of one and ten's", gone) return } dropped, err := v.DropKeyGone() - if err != nil || len(dropped) != 1 { + if err != nil || len(dropped) != 3 { t.Errorf("dropped %v, %v", dropped, err) return } - if left, _ := v.Objects.List(); len(left) != len(kept)-1 { + if left, _ := v.Objects.List(); len(left) != len(kept)-3 { t.Errorf("the drop left %d objects of %d", len(left), len(kept)) return } @@ -107,7 +108,7 @@ func TestValuesOfAnySizeThroughTheRuntime(t *testing.T) { m := newMemory() out := runtimeOf(t, m, func() (any, error) { v := ForState("views") - valueRows(t, v) + valueRows(t, v, 1, 10) // a bundle puts at most 8 MiB in an object: 100x is a process's return "done", nil }) if !strings.Contains(out.(string), "done") { @@ -156,33 +157,16 @@ func (r *raceKV) Get(key string) (*Entry, error) { return e, err } -// A store that takes no object as large as the value gets it split over objects, read back whole, replaced and dropped as -// one (a bundle's runtime carries a message of 16 MiB). -func TestAValueLargerThanTheStoreTakesIsSplitOverObjects(t *testing.T) { +// A store that takes no object as large as the value refuses it, loudly, and writes nothing. +func TestAValueLargerThanTheStoreTakesIsRefused(t *testing.T) { m := newMemory() m.most = 100_000 v := m.values() - want := big(5*kvlimit.ModuleStateValueBytes + 10) - if err := v.Put("k", want); err != nil { - t.Fatal(err) + err := v.Put("k", big(5*kvlimit.ModuleStateValueBytes)) + if err == nil || !strings.Contains(err.Error(), "refused") { + t.Fatalf("the put: %v", err) } - if l, _ := (memObjects{m}).List(); len(l) < 13 { - t.Fatalf("%d objects, want the value split over at least 13", len(l)) - } - var got map[string]string - if ok, err := v.Get("k", &got); err != nil || !ok || got["text"] != want["text"] { - t.Fatalf("read back %d, %v, %v", len(got["text"]), ok, err) - } - if err := v.Delete("k"); err != nil { - t.Fatal(err) - } - kept, _ := v.Kept() - for _, k := range kept { - if !k.KeyGone || k.Key != "k" { - t.Fatalf("a part %+v is not listed as key-gone for k", k) - } - } - if dropped, err := v.DropKeyGone(); err != nil || len(dropped) != len(kept) { - t.Fatalf("dropped %v of %d, %v", dropped, len(kept), err) + if e, _ := m.Get("k"); e != nil || len(m.objects) != 0 { + t.Fatal("a refused value left a key or an object") } }