Design 47 H1: a receiver reads its store's fill once per transfer (H-A-227, H-A-228) #305
@@ -45,9 +45,14 @@ type Transfer[M any] struct {
|
||||
// chunks taken per part.
|
||||
Held []string `json:"held,omitempty"`
|
||||
Chunks map[string]int `json:"chunks,omitempty"`
|
||||
Opened time.Time `json:"opened"`
|
||||
Last time.Time `json:"last"`
|
||||
Shape M `json:"shape"`
|
||||
// FillAtOpen is the objects' fill read from the store's figures when the transfer opened (or when chunks were dropped
|
||||
// since), and TakenSince the bytes of the chunks taken after that read: the fill told per chunk is their sum, kept
|
||||
// with the transfer, never read again per chunk (design 47 H lead answers H-A-227, H-A-228).
|
||||
FillAtOpen uint64 `json:"fill_at_open,omitempty"`
|
||||
TakenSince uint64 `json:"taken_since,omitempty"`
|
||||
Opened time.Time `json:"opened"`
|
||||
Last time.Time `json:"last"`
|
||||
Shape M `json:"shape"`
|
||||
}
|
||||
|
||||
// Lacks is the digests of the parts the transfer does not hold yet, sorted.
|
||||
@@ -197,11 +202,12 @@ func (r *Receiver[M]) Open(id, opener string, parts []Want, shape M) (Received,
|
||||
}
|
||||
return Received{ID: id, Lacks: t.Lacks()}, r.arm(id)
|
||||
}
|
||||
t := Transfer[M]{ID: id, Opener: opener, Parts: parts, Chunks: map[string]int{}, Opened: r.now(), Last: r.now(), Shape: shape}
|
||||
used, err := r.Objects.Used()
|
||||
if err != nil {
|
||||
return Received{}, fmt.Errorf("the objects' fill cannot be read: %w", err)
|
||||
}
|
||||
t := Transfer[M]{ID: id, Opener: opener, Parts: parts, Chunks: map[string]int{}, FillAtOpen: used, Opened: r.now(),
|
||||
Last: r.now(), Shape: shape}
|
||||
var size uint64
|
||||
for _, p := range parts {
|
||||
size += uint64(max(p.Size, 0))
|
||||
@@ -219,7 +225,7 @@ func (r *Receiver[M]) Open(id, opener string, parts []Want, shape M) (Received,
|
||||
if err := r.Transfers.Put(t); err != nil {
|
||||
return Received{}, fmt.Errorf("transfer %s cannot be kept: %w", id, err)
|
||||
}
|
||||
r.fill()
|
||||
r.tell(used)
|
||||
if len(t.Lacks()) == 0 {
|
||||
return Received{ID: id, Ended: true}, r.finish(t)
|
||||
}
|
||||
@@ -271,6 +277,7 @@ func (r *Receiver[M]) Take(id, opener, digest string, sequence int, last bool, d
|
||||
// A part the holder lacks travels again whole from chunk 0 (design 51 Publishing): its earlier chunks are dropped.
|
||||
r.dropChunks(t, digest)
|
||||
t.Chunks[digest] = 0
|
||||
r.reanchor(&t)
|
||||
}
|
||||
if err := r.Objects.Put(r.chunkName(id, digest, sequence), data); err != nil {
|
||||
return Received{}, fmt.Errorf("the objects did not take the chunk: %w", err)
|
||||
@@ -280,6 +287,7 @@ func (r *Receiver[M]) Take(id, opener, digest string, sequence int, last bool, d
|
||||
t.Chunks = map[string]int{}
|
||||
}
|
||||
t.Chunks[digest] = sequence + 1
|
||||
t.TakenSince += uint64(len(data))
|
||||
t.Last = r.now()
|
||||
if last {
|
||||
whole, err := r.part(t, digest)
|
||||
@@ -289,8 +297,9 @@ func (r *Receiver[M]) Take(id, opener, digest string, sequence int, last bool, d
|
||||
if got := "sha256:" + sum(whole); got != digest {
|
||||
r.dropChunks(t, digest)
|
||||
delete(t.Chunks, digest)
|
||||
r.reanchor(&t)
|
||||
_ = r.Transfers.Put(t)
|
||||
r.fill()
|
||||
r.tell(t.FillAtOpen)
|
||||
return Received{}, fmt.Errorf("refused: the bytes of part %s hash to %s; send the part again", digest, got)
|
||||
}
|
||||
t.Held = append(t.Held, digest)
|
||||
@@ -298,7 +307,7 @@ func (r *Receiver[M]) Take(id, opener, digest string, sequence int, last bool, d
|
||||
if err := r.Transfers.Put(t); err != nil {
|
||||
return Received{}, err
|
||||
}
|
||||
r.fill()
|
||||
r.tell(t.FillAtOpen + t.TakenSince)
|
||||
if len(t.Lacks()) == 0 {
|
||||
return Received{ID: id, Ended: true}, r.finish(t)
|
||||
}
|
||||
@@ -365,19 +374,36 @@ func (r *Receiver[M]) chunkName(id, digest string, seq int) string {
|
||||
return fmt.Sprintf("%s%s/%s/%d", r.Prefix, id, strings.TrimPrefix(digest, "sha256:"), seq)
|
||||
}
|
||||
|
||||
// fill tells Filled the objects' fill.
|
||||
// fill reads the objects' fill from the store's figures and tells Filled: at a transfer's finish and drop, never per
|
||||
// chunk (H-A-227).
|
||||
func (r *Receiver[M]) fill() {
|
||||
cap := r.CapNow()
|
||||
if r.Filled == nil || cap == 0 {
|
||||
if r.Filled == nil {
|
||||
return
|
||||
}
|
||||
used, err := r.Objects.Used()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
r.tell(used)
|
||||
}
|
||||
|
||||
// tell tells Filled a fill already known: read at the open, or that read plus the bytes taken since.
|
||||
func (r *Receiver[M]) tell(used uint64) {
|
||||
cap := r.CapNow()
|
||||
if r.Filled == nil || cap == 0 {
|
||||
return
|
||||
}
|
||||
r.Filled(used, cap, used*100 >= cap*r.fullPct())
|
||||
}
|
||||
|
||||
// reanchor reads the fill again after a transfer's chunks were dropped, whose sizes it does not keep: the read becomes
|
||||
// the transfer's FillAtOpen and nothing is taken since. A read that fails keeps the figures it had.
|
||||
func (r *Receiver[M]) reanchor(t *Transfer[M]) {
|
||||
if used, err := r.Objects.Used(); err == nil {
|
||||
t.FillAtOpen, t.TakenSince = used, 0
|
||||
}
|
||||
}
|
||||
|
||||
// Drop drops a transfer, by hand or because it was idle: its chunks and its key.
|
||||
func (r *Receiver[M]) Drop(id string) (Transfer[M], error) {
|
||||
t, found, err := r.Transfers.Get(id)
|
||||
|
||||
@@ -29,6 +29,14 @@ func (m memObjects) Used() (uint64, error) {
|
||||
return u, nil
|
||||
}
|
||||
|
||||
// countedObjects counts the reads of the objects' fill.
|
||||
type countedObjects struct {
|
||||
memObjects
|
||||
reads int
|
||||
}
|
||||
|
||||
func (c *countedObjects) Used() (uint64, error) { c.reads++; return c.memObjects.Used() }
|
||||
|
||||
// memTransfers keeps each transfer as the bus keeps it, its JSON, so every case reads back what a key-value state
|
||||
// gives (BusTransfers): an empty map kept with omitempty comes back nil.
|
||||
type memTransfers[M any] map[string][]byte
|
||||
@@ -244,6 +252,43 @@ func aHolderTakesAFileInParts(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
// Design 47 H lead answers H-A-227, H-A-228: the fill is read from the store once when a transfer opens and kept in the
|
||||
// transfer with the bytes taken since; each chunk tells that sum and reads nothing, also through a receiver built
|
||||
// anew per call, as the holders build it; the finish reads it again.
|
||||
t.Run("the fill is read once at the open and summed per chunk", func(t *testing.T) {
|
||||
store := memObjects{"other": make([]byte, 100)}
|
||||
counted := &countedObjects{memObjects: store}
|
||||
transfers := memTransfers[packageShape]{}
|
||||
var told []uint64
|
||||
fresh := func() *Receiver[packageShape] {
|
||||
return &Receiver[packageShape]{Objects: counted, Transfers: transfers, Cap: 1 << 20, MaxChunk: 100, Now: func() time.Time { return now },
|
||||
Filled: func(used, _ uint64, _ bool) { told = append(told, used) },
|
||||
Store: func(Transfer[packageShape], map[string][]byte) error { return nil }}
|
||||
}
|
||||
file := []byte(strings.Repeat("b", 1000))
|
||||
w := Want{Digest: digestOf(file), Size: int64(len(file))}
|
||||
if _, err := fresh().Open("t9", "builder", []Want{w}, packageShape{"m", "1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for seq := 0; seq < 9; seq++ {
|
||||
if _, err := fresh().Take("t9", "builder", w.Digest, seq, false, file[seq*100:(seq+1)*100], nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if counted.reads != 1 {
|
||||
t.Errorf("the fill was read %d times across an open and 9 chunks, want once", counted.reads)
|
||||
}
|
||||
if want := []uint64{100, 200, 300, 400, 500, 600, 700, 800, 900, 1000}; !reflect.DeepEqual(told, want) {
|
||||
t.Errorf("fills told %v, want %v (the read at the open plus the bytes taken since)", told, want)
|
||||
}
|
||||
if got, err := fresh().Take("t9", "builder", w.Digest, 9, true, file[900:], nil); err != nil || !got.Ended {
|
||||
t.Fatalf("the last chunk = %+v, %v", got, err)
|
||||
}
|
||||
if counted.reads != 2 || told[len(told)-1] != 100 {
|
||||
t.Errorf("after the finish: %d reads, told %d; want the store read again, 100", counted.reads, told[len(told)-1])
|
||||
}
|
||||
})
|
||||
|
||||
// An open transfer is bounded by its idle key (Arm); the key's end (Expired) drops it idle, arms it again for the
|
||||
// rest when it took a chunk since, and leaves a gone one; a key not written fails the open loudly. The cap the
|
||||
// binding carries (Bound) counts over Cap.
|
||||
|
||||
Reference in New Issue
Block a user