From 6a730c33e665d0cca781f249f59ce912411f202f Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 02:05:54 +0100 Subject: [PATCH] H-A-227, H-A-228: the receiver reads its objects' fill once per transfer A fill read on every chunk made the tool runner list the registry's incoming store per chunk. The kept transfer now holds the fill read at its open and the bytes taken since; each chunk tells their sum, written with the put already made per chunk, and the store is read again only at the finish, a drop, or when a part's chunks are dropped. Builds: design 47 H H-A-227, H-A-228 --- modules/mesh-delivery-sdk/parts/receive.go | 46 +++++++++++++++---- .../mesh-delivery-sdk/parts/receive_test.go | 45 ++++++++++++++++++ 2 files changed, 81 insertions(+), 10 deletions(-) diff --git a/modules/mesh-delivery-sdk/parts/receive.go b/modules/mesh-delivery-sdk/parts/receive.go index 0eef245e..5586f996 100644 --- a/modules/mesh-delivery-sdk/parts/receive.go +++ b/modules/mesh-delivery-sdk/parts/receive.go @@ -45,9 +45,14 @@ type Transfer[M any] struct { // chunks taken per part. Held []string `json:"held,omitempty"` Chunks map[string]int `json:"chunks,omitempty"` - Opened time.Time `json:"opened"` - Last time.Time `json:"last"` - Shape M `json:"shape"` + // FillAtOpen is the objects' fill read from the store's figures when the transfer opened (or when chunks were dropped + // since), and TakenSince the bytes of the chunks taken after that read: the fill told per chunk is their sum, kept + // with the transfer, never read again per chunk (design 47 H lead answers H-A-227, H-A-228). + FillAtOpen uint64 `json:"fill_at_open,omitempty"` + TakenSince uint64 `json:"taken_since,omitempty"` + Opened time.Time `json:"opened"` + Last time.Time `json:"last"` + Shape M `json:"shape"` } // Lacks is the digests of the parts the transfer does not hold yet, sorted. @@ -197,11 +202,12 @@ func (r *Receiver[M]) Open(id, opener string, parts []Want, shape M) (Received, } return Received{ID: id, Lacks: t.Lacks()}, r.arm(id) } - t := Transfer[M]{ID: id, Opener: opener, Parts: parts, Chunks: map[string]int{}, Opened: r.now(), Last: r.now(), Shape: shape} used, err := r.Objects.Used() if err != nil { return Received{}, fmt.Errorf("the objects' fill cannot be read: %w", err) } + t := Transfer[M]{ID: id, Opener: opener, Parts: parts, Chunks: map[string]int{}, FillAtOpen: used, Opened: r.now(), + Last: r.now(), Shape: shape} var size uint64 for _, p := range parts { size += uint64(max(p.Size, 0)) @@ -219,7 +225,7 @@ func (r *Receiver[M]) Open(id, opener string, parts []Want, shape M) (Received, if err := r.Transfers.Put(t); err != nil { return Received{}, fmt.Errorf("transfer %s cannot be kept: %w", id, err) } - r.fill() + r.tell(used) if len(t.Lacks()) == 0 { return Received{ID: id, Ended: true}, r.finish(t) } @@ -271,6 +277,7 @@ func (r *Receiver[M]) Take(id, opener, digest string, sequence int, last bool, d // A part the holder lacks travels again whole from chunk 0 (design 51 Publishing): its earlier chunks are dropped. r.dropChunks(t, digest) t.Chunks[digest] = 0 + r.reanchor(&t) } if err := r.Objects.Put(r.chunkName(id, digest, sequence), data); err != nil { return Received{}, fmt.Errorf("the objects did not take the chunk: %w", err) @@ -280,6 +287,7 @@ func (r *Receiver[M]) Take(id, opener, digest string, sequence int, last bool, d t.Chunks = map[string]int{} } t.Chunks[digest] = sequence + 1 + t.TakenSince += uint64(len(data)) t.Last = r.now() if last { whole, err := r.part(t, digest) @@ -289,8 +297,9 @@ func (r *Receiver[M]) Take(id, opener, digest string, sequence int, last bool, d if got := "sha256:" + sum(whole); got != digest { r.dropChunks(t, digest) delete(t.Chunks, digest) + r.reanchor(&t) _ = r.Transfers.Put(t) - r.fill() + r.tell(t.FillAtOpen) return Received{}, fmt.Errorf("refused: the bytes of part %s hash to %s; send the part again", digest, got) } t.Held = append(t.Held, digest) @@ -298,7 +307,7 @@ func (r *Receiver[M]) Take(id, opener, digest string, sequence int, last bool, d if err := r.Transfers.Put(t); err != nil { return Received{}, err } - r.fill() + r.tell(t.FillAtOpen + t.TakenSince) if len(t.Lacks()) == 0 { return Received{ID: id, Ended: true}, r.finish(t) } @@ -365,19 +374,36 @@ func (r *Receiver[M]) chunkName(id, digest string, seq int) string { return fmt.Sprintf("%s%s/%s/%d", r.Prefix, id, strings.TrimPrefix(digest, "sha256:"), seq) } -// fill tells Filled the objects' fill. +// fill reads the objects' fill from the store's figures and tells Filled: at a transfer's finish and drop, never per +// chunk (H-A-227). func (r *Receiver[M]) fill() { - cap := r.CapNow() - if r.Filled == nil || cap == 0 { + if r.Filled == nil { return } used, err := r.Objects.Used() if err != nil { return } + r.tell(used) +} + +// tell tells Filled a fill already known: read at the open, or that read plus the bytes taken since. +func (r *Receiver[M]) tell(used uint64) { + cap := r.CapNow() + if r.Filled == nil || cap == 0 { + return + } r.Filled(used, cap, used*100 >= cap*r.fullPct()) } +// reanchor reads the fill again after a transfer's chunks were dropped, whose sizes it does not keep: the read becomes +// the transfer's FillAtOpen and nothing is taken since. A read that fails keeps the figures it had. +func (r *Receiver[M]) reanchor(t *Transfer[M]) { + if used, err := r.Objects.Used(); err == nil { + t.FillAtOpen, t.TakenSince = used, 0 + } +} + // Drop drops a transfer, by hand or because it was idle: its chunks and its key. func (r *Receiver[M]) Drop(id string) (Transfer[M], error) { t, found, err := r.Transfers.Get(id) diff --git a/modules/mesh-delivery-sdk/parts/receive_test.go b/modules/mesh-delivery-sdk/parts/receive_test.go index 54046826..254dc48b 100644 --- a/modules/mesh-delivery-sdk/parts/receive_test.go +++ b/modules/mesh-delivery-sdk/parts/receive_test.go @@ -29,6 +29,14 @@ func (m memObjects) Used() (uint64, error) { return u, nil } +// countedObjects counts the reads of the objects' fill. +type countedObjects struct { + memObjects + reads int +} + +func (c *countedObjects) Used() (uint64, error) { c.reads++; return c.memObjects.Used() } + // memTransfers keeps each transfer as the bus keeps it, its JSON, so every case reads back what a key-value state // gives (BusTransfers): an empty map kept with omitempty comes back nil. type memTransfers[M any] map[string][]byte @@ -244,6 +252,43 @@ func aHolderTakesAFileInParts(t *testing.T) { } }) + // Design 47 H lead answers H-A-227, H-A-228: the fill is read from the store once when a transfer opens and kept in the + // transfer with the bytes taken since; each chunk tells that sum and reads nothing, also through a receiver built + // anew per call, as the holders build it; the finish reads it again. + t.Run("the fill is read once at the open and summed per chunk", func(t *testing.T) { + store := memObjects{"other": make([]byte, 100)} + counted := &countedObjects{memObjects: store} + transfers := memTransfers[packageShape]{} + var told []uint64 + fresh := func() *Receiver[packageShape] { + return &Receiver[packageShape]{Objects: counted, Transfers: transfers, Cap: 1 << 20, MaxChunk: 100, Now: func() time.Time { return now }, + Filled: func(used, _ uint64, _ bool) { told = append(told, used) }, + Store: func(Transfer[packageShape], map[string][]byte) error { return nil }} + } + file := []byte(strings.Repeat("b", 1000)) + w := Want{Digest: digestOf(file), Size: int64(len(file))} + if _, err := fresh().Open("t9", "builder", []Want{w}, packageShape{"m", "1"}); err != nil { + t.Fatal(err) + } + for seq := 0; seq < 9; seq++ { + if _, err := fresh().Take("t9", "builder", w.Digest, seq, false, file[seq*100:(seq+1)*100], nil); err != nil { + t.Fatal(err) + } + } + if counted.reads != 1 { + t.Errorf("the fill was read %d times across an open and 9 chunks, want once", counted.reads) + } + if want := []uint64{100, 200, 300, 400, 500, 600, 700, 800, 900, 1000}; !reflect.DeepEqual(told, want) { + t.Errorf("fills told %v, want %v (the read at the open plus the bytes taken since)", told, want) + } + if got, err := fresh().Take("t9", "builder", w.Digest, 9, true, file[900:], nil); err != nil || !got.Ended { + t.Fatalf("the last chunk = %+v, %v", got, err) + } + if counted.reads != 2 || told[len(told)-1] != 100 { + t.Errorf("after the finish: %d reads, told %d; want the store read again, 100", counted.reads, told[len(told)-1]) + } + }) + // An open transfer is bounded by its idle key (Arm); the key's end (Expired) drops it idle, arms it again for the // rest when it took a chunk since, and leaves a gone one; a key not written fails the open loudly. The cap the // binding carries (Bound) counts over Cap.