From e2eb99fc19de6509d61c6e5beef5fc1563643a05 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 13 Nov 2026 22:04:21 +0100 Subject: [PATCH] A-373 grants-list-and-pass-as-objects: the keys list and pass of user-list name an object of the part's bucket, since each outgrows one bucket value Builds: design 47 G A-373 (G-22-lead-answers-2.md) Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- modules/mesh-grants/cmd/mesh-grants/main.go | 2 +- modules/mesh-grants/cmd/mesh-grants/sealed.go | 11 +++ modules/mesh-grants/cmd/mesh-grants/tools.go | 2 +- .../mesh-grants/internal/grants/bigvalues.go | 76 +++++++++++++++++++ .../internal/grants/bigvalues_test.go | 65 ++++++++++++++++ 5 files changed, 154 insertions(+), 2 deletions(-) create mode 100644 modules/mesh-grants/internal/grants/bigvalues.go create mode 100644 modules/mesh-grants/internal/grants/bigvalues_test.go diff --git a/modules/mesh-grants/cmd/mesh-grants/main.go b/modules/mesh-grants/cmd/mesh-grants/main.go index 31f13c01..7c7c68af 100644 --- a/modules/mesh-grants/cmd/mesh-grants/main.go +++ b/modules/mesh-grants/cmd/mesh-grants/main.go @@ -177,7 +177,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err } fv.Assignment = assignment - f.Part = &grants.Part{Read: fv, Call: conn, Store: ownStore{conn}, Emit: emitting{conn}, Views: fv, FirstViewContext: ctx, + f.Part = &grants.Part{Read: fv, Call: conn, Store: grants.BigValues(ownStore{conn}, sealedObjects{ctx: ctx, js: js}), Emit: emitting{conn}, Views: fv, FirstViewContext: ctx, Create: creating{ctx: ctx, js: js}, Consumers: creating{ctx: ctx, js: js}, NamedPurge: creating{ctx: ctx, js: js}, Objects: sealedObjects{ctx: ctx, js: js}, Purging: ownPurging{ctx: ctx, js: js, module: account.Module}, Logf: logger.Printf, Bus: grants.BusAddress{URL: account.URL, Fingerprint: account.Fingerprint}} diff --git a/modules/mesh-grants/cmd/mesh-grants/sealed.go b/modules/mesh-grants/cmd/mesh-grants/sealed.go index ab0cf73b..d1d16683 100644 --- a/modules/mesh-grants/cmd/mesh-grants/sealed.go +++ b/modules/mesh-grants/cmd/mesh-grants/sealed.go @@ -53,3 +53,14 @@ func (o sealedObjects) DeleteObject(name string) error { } return nil } + +// GetObject reads one object of the bucket whole: what the keys `list` and `pass` of the state user-list name (design 47 G A-373). +func (o sealedObjects) GetObject(name string) ([]byte, error) { + ctx, cancel := context.WithTimeout(o.ctx, requestWithin) + defer cancel() + s, err := o.store(ctx) + if err != nil { + return nil, err + } + return s.GetBytes(ctx, name) +} diff --git a/modules/mesh-grants/cmd/mesh-grants/tools.go b/modules/mesh-grants/cmd/mesh-grants/tools.go index c71b8a54..94955e71 100644 --- a/modules/mesh-grants/cmd/mesh-grants/tools.go +++ b/modules/mesh-grants/cmd/mesh-grants/tools.go @@ -320,7 +320,7 @@ func recordHandActAs(cause string) func(what, why string) error { func serveTools() error { tv := &toolViews{} gate := newSeatGate() - p := &grants.Part{Read: runtimeReader{views: tv}, Call: runtimeBus{}, Store: runtimeStore{}, Emit: runtimeBus{}, SeatList: gate} + p := &grants.Part{Read: runtimeReader{views: tv}, Call: runtimeBus{}, Store: grants.BigValues(runtimeStore{}, nil), Emit: runtimeBus{}, SeatList: gate} // The reader asks through the runtime, which serves once Serve has its channel. go func() { <-stdio.Served() diff --git a/modules/mesh-grants/internal/grants/bigvalues.go b/modules/mesh-grants/internal/grants/bigvalues.go new file mode 100644 index 00000000..b29c71c4 --- /dev/null +++ b/modules/mesh-grants/internal/grants/bigvalues.go @@ -0,0 +1,76 @@ +package grants + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" +) + +// **The part's user list and pass result are kept as objects** (design 47 G A-373): each outgrows one value of a key-value bucket +// (live, 2026-11-13: the list 422584 bytes and the pass 479989, over the 262144 the bucket takes), so the state `user-list` keeps +// them in the part's object bucket and the keys `list` and `pass` name the object, with its digest and size. A write that cannot be +// made stays an error, as it was; a pointer that cannot be followed is an error naming the object, never an empty answer. + +// BigObjects is the object bucket the keys' values are kept in: the part's own (the process's connection). +type BigObjects interface { + PutObject(name string, data []byte) error + GetObject(name string) ([]byte, error) +} + +// bigKeys are the keys of StateUserList kept as objects. +var bigKeys = map[string]bool{KeyList: true, KeyPass: true} + +// BigObjectName is the object a key of StateUserList is kept in. +func BigObjectName(key string) string { return "state." + StateUserList + "." + key } + +// bigPointer is what the key holds in place of the value. +type bigPointer struct { + KeptIn string `json:"kept_in_object"` + SHA256 string `json:"sha256"` + Bytes int `json:"bytes"` +} + +// BigValues is store with the keys `list` and `pass` of the state user-list kept in objects. With no objects (the tools') a value is +// kept in the key whole, and a key that names an object is an error saying where the value is. +func BigValues(store Store, objects BigObjects) Store { return bigStore{Store: store, objects: objects} } + +type bigStore struct { + Store + objects BigObjects +} + +func (s bigStore) Put(state, key string, value []byte) (uint64, error) { + if state != StateUserList || !bigKeys[key] || s.objects == nil { + return s.Store.Put(state, key, value) + } + name := BigObjectName(key) + if err := s.objects.PutObject(name, value); err != nil { + return 0, fmt.Errorf("the value of %s could not be put as the object %s: %w", key, name, err) + } + sum := sha256.Sum256(value) + ptr, _ := json.Marshal(bigPointer{KeptIn: name, SHA256: hex.EncodeToString(sum[:]), Bytes: len(value)}) + return s.Store.Put(state, key, ptr) +} + +func (s bigStore) Get(state, key string) ([]byte, uint64, bool, error) { + raw, rev, found, err := s.Store.Get(state, key) + if err != nil || !found || state != StateUserList || !bigKeys[key] { + return raw, rev, found, err + } + var p bigPointer + if json.Unmarshal(raw, &p) != nil || p.KeptIn == "" { + return raw, rev, found, nil // a value kept whole, as before + } + if s.objects == nil { + return nil, rev, true, fmt.Errorf("the value of %s is kept in the object %s of the part's object bucket, which this reader cannot open", key, p.KeptIn) + } + data, err := s.objects.GetObject(p.KeptIn) + if err != nil { + return nil, rev, true, fmt.Errorf("the object %s that %s names cannot be read: %w", p.KeptIn, key, err) + } + if sum := sha256.Sum256(data); hex.EncodeToString(sum[:]) != p.SHA256 { + return nil, rev, true, fmt.Errorf("the object %s is not the one %s names (digest differs): it was replaced while read", p.KeptIn, key) + } + return data, rev, true, nil +} diff --git a/modules/mesh-grants/internal/grants/bigvalues_test.go b/modules/mesh-grants/internal/grants/bigvalues_test.go new file mode 100644 index 00000000..7b83f467 --- /dev/null +++ b/modules/mesh-grants/internal/grants/bigvalues_test.go @@ -0,0 +1,65 @@ +package grants + +import ( + "bytes" + "errors" + "strings" + "testing" +) + +// limitStore refuses a value over the bucket's 262144 bytes, as the bus does. +type limitStore struct{ memStore } + +func (l limitStore) Put(state, key string, value []byte) (uint64, error) { + if len(value) > 262144 { + return 0, errors.New("the value is over the 262144 bytes the bucket takes") + } + return l.memStore.Put(state, key, value) +} + +type memObjects map[string][]byte + +func (m memObjects) PutObject(name string, data []byte) error { m[name] = data; return nil } +func (m memObjects) GetObject(name string) ([]byte, error) { + if d, ok := m[name]; ok { + return d, nil + } + return nil, errors.New("no such object") +} + +// Design 47 G A-373: the user list and the pass result, over 256 KB each, are written and read back through objects the keys name. +func TestAListOverTheBucketsValueIsKeptAsAnObject(t *testing.T) { + big := []byte(`{"v":"` + strings.Repeat("x", 450000) + `"}`) + for _, key := range []string{KeyList, KeyPass} { + raw := limitStore{memStore{}} + if _, err := raw.Put(StateUserList, key, big); err == nil { + t.Fatal("the bucket takes a value over its limit") + } + objects := memObjects{} + s := BigValues(raw, objects) + if _, err := s.Put(StateUserList, key, big); err != nil { + t.Fatalf("%s: %v", key, err) + } + got, _, found, err := s.Get(StateUserList, key) + if err != nil || !found || !bytes.Equal(got, big) { + t.Fatalf("%s read back: %d bytes, found %v, %v", key, len(got), found, err) + } + // The tools' reader holds no objects: it is told where the value is, never given an empty one. + if _, _, _, err := BigValues(raw, nil).Get(StateUserList, key); err == nil || !strings.Contains(err.Error(), BigObjectName(key)) { + t.Errorf("%s read with no objects: %v", key, err) + } + // A write that cannot be made stays an error. + if _, err := BigValues(raw, failingObjects{}).Put(StateUserList, key, big); err == nil { + t.Errorf("%s: an object that could not be put is no error", key) + } + // Another key and a small value are kept as they were. + small := []byte(`{"a":1}`) + if _, err := s.Put(StateUserList, KeyAsked, small); err != nil || len(objects) != 1 { + t.Errorf("another key: %v, objects %v", err, objects) + } + } +} + +type failingObjects struct{ memObjects } + +func (failingObjects) PutObject(string, []byte) error { return errors.New("refused") }