diff --git a/modules/mesh-delivery-sdk/parts/receive.go b/modules/mesh-delivery-sdk/parts/receive.go index 0eef245e..5586f996 100644 --- a/modules/mesh-delivery-sdk/parts/receive.go +++ b/modules/mesh-delivery-sdk/parts/receive.go @@ -45,9 +45,14 @@ type Transfer[M any] struct { // chunks taken per part. Held []string `json:"held,omitempty"` Chunks map[string]int `json:"chunks,omitempty"` - Opened time.Time `json:"opened"` - Last time.Time `json:"last"` - Shape M `json:"shape"` + // FillAtOpen is the objects' fill read from the store's figures when the transfer opened (or when chunks were dropped + // since), and TakenSince the bytes of the chunks taken after that read: the fill told per chunk is their sum, kept + // with the transfer, never read again per chunk (design 47 H lead answers H-A-227, H-A-228). + FillAtOpen uint64 `json:"fill_at_open,omitempty"` + TakenSince uint64 `json:"taken_since,omitempty"` + Opened time.Time `json:"opened"` + Last time.Time `json:"last"` + Shape M `json:"shape"` } // Lacks is the digests of the parts the transfer does not hold yet, sorted. @@ -197,11 +202,12 @@ func (r *Receiver[M]) Open(id, opener string, parts []Want, shape M) (Received, } return Received{ID: id, Lacks: t.Lacks()}, r.arm(id) } - t := Transfer[M]{ID: id, Opener: opener, Parts: parts, Chunks: map[string]int{}, Opened: r.now(), Last: r.now(), Shape: shape} used, err := r.Objects.Used() if err != nil { return Received{}, fmt.Errorf("the objects' fill cannot be read: %w", err) } + t := Transfer[M]{ID: id, Opener: opener, Parts: parts, Chunks: map[string]int{}, FillAtOpen: used, Opened: r.now(), + Last: r.now(), Shape: shape} var size uint64 for _, p := range parts { size += uint64(max(p.Size, 0)) @@ -219,7 +225,7 @@ func (r *Receiver[M]) Open(id, opener string, parts []Want, shape M) (Received, if err := r.Transfers.Put(t); err != nil { return Received{}, fmt.Errorf("transfer %s cannot be kept: %w", id, err) } - r.fill() + r.tell(used) if len(t.Lacks()) == 0 { return Received{ID: id, Ended: true}, r.finish(t) } @@ -271,6 +277,7 @@ func (r *Receiver[M]) Take(id, opener, digest string, sequence int, last bool, d // A part the holder lacks travels again whole from chunk 0 (design 51 Publishing): its earlier chunks are dropped. r.dropChunks(t, digest) t.Chunks[digest] = 0 + r.reanchor(&t) } if err := r.Objects.Put(r.chunkName(id, digest, sequence), data); err != nil { return Received{}, fmt.Errorf("the objects did not take the chunk: %w", err) @@ -280,6 +287,7 @@ func (r *Receiver[M]) Take(id, opener, digest string, sequence int, last bool, d t.Chunks = map[string]int{} } t.Chunks[digest] = sequence + 1 + t.TakenSince += uint64(len(data)) t.Last = r.now() if last { whole, err := r.part(t, digest) @@ -289,8 +297,9 @@ func (r *Receiver[M]) Take(id, opener, digest string, sequence int, last bool, d if got := "sha256:" + sum(whole); got != digest { r.dropChunks(t, digest) delete(t.Chunks, digest) + r.reanchor(&t) _ = r.Transfers.Put(t) - r.fill() + r.tell(t.FillAtOpen) return Received{}, fmt.Errorf("refused: the bytes of part %s hash to %s; send the part again", digest, got) } t.Held = append(t.Held, digest) @@ -298,7 +307,7 @@ func (r *Receiver[M]) Take(id, opener, digest string, sequence int, last bool, d if err := r.Transfers.Put(t); err != nil { return Received{}, err } - r.fill() + r.tell(t.FillAtOpen + t.TakenSince) if len(t.Lacks()) == 0 { return Received{ID: id, Ended: true}, r.finish(t) } @@ -365,19 +374,36 @@ func (r *Receiver[M]) chunkName(id, digest string, seq int) string { return fmt.Sprintf("%s%s/%s/%d", r.Prefix, id, strings.TrimPrefix(digest, "sha256:"), seq) } -// fill tells Filled the objects' fill. +// fill reads the objects' fill from the store's figures and tells Filled: at a transfer's finish and drop, never per +// chunk (H-A-227). func (r *Receiver[M]) fill() { - cap := r.CapNow() - if r.Filled == nil || cap == 0 { + if r.Filled == nil { return } used, err := r.Objects.Used() if err != nil { return } + r.tell(used) +} + +// tell tells Filled a fill already known: read at the open, or that read plus the bytes taken since. +func (r *Receiver[M]) tell(used uint64) { + cap := r.CapNow() + if r.Filled == nil || cap == 0 { + return + } r.Filled(used, cap, used*100 >= cap*r.fullPct()) } +// reanchor reads the fill again after a transfer's chunks were dropped, whose sizes it does not keep: the read becomes +// the transfer's FillAtOpen and nothing is taken since. A read that fails keeps the figures it had. +func (r *Receiver[M]) reanchor(t *Transfer[M]) { + if used, err := r.Objects.Used(); err == nil { + t.FillAtOpen, t.TakenSince = used, 0 + } +} + // Drop drops a transfer, by hand or because it was idle: its chunks and its key. func (r *Receiver[M]) Drop(id string) (Transfer[M], error) { t, found, err := r.Transfers.Get(id) diff --git a/modules/mesh-delivery-sdk/parts/receive_test.go b/modules/mesh-delivery-sdk/parts/receive_test.go index 54046826..254dc48b 100644 --- a/modules/mesh-delivery-sdk/parts/receive_test.go +++ b/modules/mesh-delivery-sdk/parts/receive_test.go @@ -29,6 +29,14 @@ func (m memObjects) Used() (uint64, error) { return u, nil } +// countedObjects counts the reads of the objects' fill. +type countedObjects struct { + memObjects + reads int +} + +func (c *countedObjects) Used() (uint64, error) { c.reads++; return c.memObjects.Used() } + // memTransfers keeps each transfer as the bus keeps it, its JSON, so every case reads back what a key-value state // gives (BusTransfers): an empty map kept with omitempty comes back nil. type memTransfers[M any] map[string][]byte @@ -244,6 +252,43 @@ func aHolderTakesAFileInParts(t *testing.T) { } }) + // Design 47 H lead answers H-A-227, H-A-228: the fill is read from the store once when a transfer opens and kept in the + // transfer with the bytes taken since; each chunk tells that sum and reads nothing, also through a receiver built + // anew per call, as the holders build it; the finish reads it again. + t.Run("the fill is read once at the open and summed per chunk", func(t *testing.T) { + store := memObjects{"other": make([]byte, 100)} + counted := &countedObjects{memObjects: store} + transfers := memTransfers[packageShape]{} + var told []uint64 + fresh := func() *Receiver[packageShape] { + return &Receiver[packageShape]{Objects: counted, Transfers: transfers, Cap: 1 << 20, MaxChunk: 100, Now: func() time.Time { return now }, + Filled: func(used, _ uint64, _ bool) { told = append(told, used) }, + Store: func(Transfer[packageShape], map[string][]byte) error { return nil }} + } + file := []byte(strings.Repeat("b", 1000)) + w := Want{Digest: digestOf(file), Size: int64(len(file))} + if _, err := fresh().Open("t9", "builder", []Want{w}, packageShape{"m", "1"}); err != nil { + t.Fatal(err) + } + for seq := 0; seq < 9; seq++ { + if _, err := fresh().Take("t9", "builder", w.Digest, seq, false, file[seq*100:(seq+1)*100], nil); err != nil { + t.Fatal(err) + } + } + if counted.reads != 1 { + t.Errorf("the fill was read %d times across an open and 9 chunks, want once", counted.reads) + } + if want := []uint64{100, 200, 300, 400, 500, 600, 700, 800, 900, 1000}; !reflect.DeepEqual(told, want) { + t.Errorf("fills told %v, want %v (the read at the open plus the bytes taken since)", told, want) + } + if got, err := fresh().Take("t9", "builder", w.Digest, 9, true, file[900:], nil); err != nil || !got.Ended { + t.Fatalf("the last chunk = %+v, %v", got, err) + } + if counted.reads != 2 || told[len(told)-1] != 100 { + t.Errorf("after the finish: %d reads, told %d; want the store read again, 100", counted.reads, told[len(told)-1]) + } + }) + // An open transfer is bounded by its idle key (Arm); the key's end (Expired) drops it idle, arms it again for the // rest when it took a chunk since, and leaves a gone one; a key not written fails the open loudly. The cap the // binding carries (Bound) counts over Cap.