Merge pull request 'Design 47 H1: a receiver reads its store's fill once per transfer (H-A-227, H-A-228)' (#305) from fix/47-H1-receiver-fill-once into main

This commit was merged in pull request #305.
This commit is contained in:
2026-11-21 01:12:32 +00:00
2 changed files with 81 additions and 10 deletions
+36 -10
View File
@@ -45,9 +45,14 @@ type Transfer[M any] struct {
// chunks taken per part.
Held []string `json:"held,omitempty"`
Chunks map[string]int `json:"chunks,omitempty"`
Opened time.Time `json:"opened"`
Last time.Time `json:"last"`
Shape M `json:"shape"`
// FillAtOpen is the objects' fill read from the store's figures when the transfer opened (or when chunks were dropped
// since), and TakenSince the bytes of the chunks taken after that read: the fill told per chunk is their sum, kept
// with the transfer, never read again per chunk (design 47 H lead answers H-A-227, H-A-228).
FillAtOpen uint64 `json:"fill_at_open,omitempty"`
TakenSince uint64 `json:"taken_since,omitempty"`
Opened time.Time `json:"opened"`
Last time.Time `json:"last"`
Shape M `json:"shape"`
}
// Lacks is the digests of the parts the transfer does not hold yet, sorted.
@@ -197,11 +202,12 @@ func (r *Receiver[M]) Open(id, opener string, parts []Want, shape M) (Received,
}
return Received{ID: id, Lacks: t.Lacks()}, r.arm(id)
}
t := Transfer[M]{ID: id, Opener: opener, Parts: parts, Chunks: map[string]int{}, Opened: r.now(), Last: r.now(), Shape: shape}
used, err := r.Objects.Used()
if err != nil {
return Received{}, fmt.Errorf("the objects' fill cannot be read: %w", err)
}
t := Transfer[M]{ID: id, Opener: opener, Parts: parts, Chunks: map[string]int{}, FillAtOpen: used, Opened: r.now(),
Last: r.now(), Shape: shape}
var size uint64
for _, p := range parts {
size += uint64(max(p.Size, 0))
@@ -219,7 +225,7 @@ func (r *Receiver[M]) Open(id, opener string, parts []Want, shape M) (Received,
if err := r.Transfers.Put(t); err != nil {
return Received{}, fmt.Errorf("transfer %s cannot be kept: %w", id, err)
}
r.fill()
r.tell(used)
if len(t.Lacks()) == 0 {
return Received{ID: id, Ended: true}, r.finish(t)
}
@@ -271,6 +277,7 @@ func (r *Receiver[M]) Take(id, opener, digest string, sequence int, last bool, d
// A part the holder lacks travels again whole from chunk 0 (design 51 Publishing): its earlier chunks are dropped.
r.dropChunks(t, digest)
t.Chunks[digest] = 0
r.reanchor(&t)
}
if err := r.Objects.Put(r.chunkName(id, digest, sequence), data); err != nil {
return Received{}, fmt.Errorf("the objects did not take the chunk: %w", err)
@@ -280,6 +287,7 @@ func (r *Receiver[M]) Take(id, opener, digest string, sequence int, last bool, d
t.Chunks = map[string]int{}
}
t.Chunks[digest] = sequence + 1
t.TakenSince += uint64(len(data))
t.Last = r.now()
if last {
whole, err := r.part(t, digest)
@@ -289,8 +297,9 @@ func (r *Receiver[M]) Take(id, opener, digest string, sequence int, last bool, d
if got := "sha256:" + sum(whole); got != digest {
r.dropChunks(t, digest)
delete(t.Chunks, digest)
r.reanchor(&t)
_ = r.Transfers.Put(t)
r.fill()
r.tell(t.FillAtOpen)
return Received{}, fmt.Errorf("refused: the bytes of part %s hash to %s; send the part again", digest, got)
}
t.Held = append(t.Held, digest)
@@ -298,7 +307,7 @@ func (r *Receiver[M]) Take(id, opener, digest string, sequence int, last bool, d
if err := r.Transfers.Put(t); err != nil {
return Received{}, err
}
r.fill()
r.tell(t.FillAtOpen + t.TakenSince)
if len(t.Lacks()) == 0 {
return Received{ID: id, Ended: true}, r.finish(t)
}
@@ -365,19 +374,36 @@ func (r *Receiver[M]) chunkName(id, digest string, seq int) string {
return fmt.Sprintf("%s%s/%s/%d", r.Prefix, id, strings.TrimPrefix(digest, "sha256:"), seq)
}
// fill tells Filled the objects' fill.
// fill reads the objects' fill from the store's figures and tells Filled: at a transfer's finish and drop, never per
// chunk (H-A-227).
func (r *Receiver[M]) fill() {
cap := r.CapNow()
if r.Filled == nil || cap == 0 {
if r.Filled == nil {
return
}
used, err := r.Objects.Used()
if err != nil {
return
}
r.tell(used)
}
// tell tells Filled a fill already known: read at the open, or that read plus the bytes taken since.
func (r *Receiver[M]) tell(used uint64) {
cap := r.CapNow()
if r.Filled == nil || cap == 0 {
return
}
r.Filled(used, cap, used*100 >= cap*r.fullPct())
}
// reanchor reads the fill again after a transfer's chunks were dropped, whose sizes it does not keep: the read becomes
// the transfer's FillAtOpen and nothing is taken since. A read that fails keeps the figures it had.
func (r *Receiver[M]) reanchor(t *Transfer[M]) {
if used, err := r.Objects.Used(); err == nil {
t.FillAtOpen, t.TakenSince = used, 0
}
}
// Drop drops a transfer, by hand or because it was idle: its chunks and its key.
func (r *Receiver[M]) Drop(id string) (Transfer[M], error) {
t, found, err := r.Transfers.Get(id)
@@ -29,6 +29,14 @@ func (m memObjects) Used() (uint64, error) {
return u, nil
}
// countedObjects counts the reads of the objects' fill.
type countedObjects struct {
memObjects
reads int
}
func (c *countedObjects) Used() (uint64, error) { c.reads++; return c.memObjects.Used() }
// memTransfers keeps each transfer as the bus keeps it, its JSON, so every case reads back what a key-value state
// gives (BusTransfers): an empty map kept with omitempty comes back nil.
type memTransfers[M any] map[string][]byte
@@ -244,6 +252,43 @@ func aHolderTakesAFileInParts(t *testing.T) {
}
})
// Design 47 H lead answers H-A-227, H-A-228: the fill is read from the store once when a transfer opens and kept in the
// transfer with the bytes taken since; each chunk tells that sum and reads nothing, also through a receiver built
// anew per call, as the holders build it; the finish reads it again.
t.Run("the fill is read once at the open and summed per chunk", func(t *testing.T) {
store := memObjects{"other": make([]byte, 100)}
counted := &countedObjects{memObjects: store}
transfers := memTransfers[packageShape]{}
var told []uint64
fresh := func() *Receiver[packageShape] {
return &Receiver[packageShape]{Objects: counted, Transfers: transfers, Cap: 1 << 20, MaxChunk: 100, Now: func() time.Time { return now },
Filled: func(used, _ uint64, _ bool) { told = append(told, used) },
Store: func(Transfer[packageShape], map[string][]byte) error { return nil }}
}
file := []byte(strings.Repeat("b", 1000))
w := Want{Digest: digestOf(file), Size: int64(len(file))}
if _, err := fresh().Open("t9", "builder", []Want{w}, packageShape{"m", "1"}); err != nil {
t.Fatal(err)
}
for seq := 0; seq < 9; seq++ {
if _, err := fresh().Take("t9", "builder", w.Digest, seq, false, file[seq*100:(seq+1)*100], nil); err != nil {
t.Fatal(err)
}
}
if counted.reads != 1 {
t.Errorf("the fill was read %d times across an open and 9 chunks, want once", counted.reads)
}
if want := []uint64{100, 200, 300, 400, 500, 600, 700, 800, 900, 1000}; !reflect.DeepEqual(told, want) {
t.Errorf("fills told %v, want %v (the read at the open plus the bytes taken since)", told, want)
}
if got, err := fresh().Take("t9", "builder", w.Digest, 9, true, file[900:], nil); err != nil || !got.Ended {
t.Fatalf("the last chunk = %+v, %v", got, err)
}
if counted.reads != 2 || told[len(told)-1] != 100 {
t.Errorf("after the finish: %d reads, told %d; want the store read again, 100", counted.reads, told[len(told)-1])
}
})
// An open transfer is bounded by its idle key (Arm); the key's end (Expired) drops it idle, arms it again for the
// rest when it took a chunk since, and leaves a gone one; a key not written fails the open loudly. The cap the
// binding carries (Bound) counts over Cap.