A transfer is finished or taken over, never left open (H-A-481)
An opener that opened a transfer again with nothing lacked was answered "nothing lacked" and the transfer never finished; another caller with the same digest was refused; and a transfer lacking nothing at its open was finished without its idle bound, so a restart during the finish left it open. Now a reopen with nothing lacked finishes it, by its opener or by another caller; a transfer that lacks a part is taken over by another caller once its last part is older than two minutes; and the idle bound is armed before every finish at an open. Builds: design 47 H2 H-A-481
This commit is contained in:
@@ -27,6 +27,10 @@ const (
|
||||
IdleAfter = 24 * time.Hour
|
||||
// FullPercent is the fill of the objects past which the receiver says they fill.
|
||||
FullPercent = 80
|
||||
// TakeOverAfter is how long a transfer's last part must lie behind it before another caller opening it with the
|
||||
// same digest takes it over (design 47 H2 H-A-481): a sender restarted mid-transfer goes on, a live one is not
|
||||
// pushed aside.
|
||||
TakeOverAfter = 2 * time.Minute
|
||||
)
|
||||
|
||||
// Want is one part an opening names: its digest (sha256:<hex>) and its size.
|
||||
@@ -180,8 +184,11 @@ func (r *Receiver[M]) fullPct() uint64 {
|
||||
}
|
||||
|
||||
// Open opens a transfer for opener, or opens it again: a transfer already open answers what it lacks, so a cut transfer
|
||||
// goes on from there, and only its opener may open it again. A new transfer that would pass the cap is refused; parts
|
||||
// the holder holds count as held; when nothing is lacked the transfer ends at once.
|
||||
// goes on from there. A transfer is finished or taken over, never left open (design 47 H2 H-A-481): one opened again
|
||||
// with nothing lacked (its finish was cut) is finished, by its opener or by another caller with the same digest; one
|
||||
// that lacks a part is another caller's only once its last part is older than TakeOverAfter, and is refused to it
|
||||
// before. A new transfer that would pass the cap is refused; parts the holder holds count as held; when nothing is
|
||||
// lacked the transfer ends at once, its idle bound armed before the finish, so a cut finish never leaves it open.
|
||||
func (r *Receiver[M]) Open(id, opener string, parts []Want, shape M) (Received, error) {
|
||||
switch {
|
||||
case strings.TrimSpace(id) == "" || strings.ContainsAny(id, "/. *>"):
|
||||
@@ -197,8 +204,17 @@ func (r *Receiver[M]) Open(id, opener string, parts []Want, shape M) (Received,
|
||||
if t, found, err := r.Transfers.Get(id); err != nil {
|
||||
return Received{}, err
|
||||
} else if found {
|
||||
if t.Opener != opener {
|
||||
return Received{}, fmt.Errorf("refused for %s: transfer %s was opened by another caller", opener, id)
|
||||
switch {
|
||||
case len(t.Lacks()) == 0:
|
||||
return r.finishArmed(t)
|
||||
case t.Opener != opener && r.now().Sub(t.Last) < TakeOverAfter:
|
||||
return Received{}, fmt.Errorf("refused for %s: transfer %s was opened by another caller, whose last part came %s ago (taken over after %s)",
|
||||
opener, id, r.now().Sub(t.Last).Round(time.Second), TakeOverAfter)
|
||||
case t.Opener != opener:
|
||||
t.Opener = opener
|
||||
if err := r.Transfers.Put(t); err != nil {
|
||||
return Received{}, fmt.Errorf("transfer %s cannot be taken over by %s: %w", id, opener, err)
|
||||
}
|
||||
}
|
||||
return Received{ID: id, Lacks: t.Lacks()}, r.arm(id)
|
||||
}
|
||||
@@ -227,11 +243,21 @@ func (r *Receiver[M]) Open(id, opener string, parts []Want, shape M) (Received,
|
||||
}
|
||||
r.tell(used)
|
||||
if len(t.Lacks()) == 0 {
|
||||
return Received{ID: id, Ended: true}, r.finish(t)
|
||||
return r.finishArmed(t)
|
||||
}
|
||||
return Received{ID: id, Lacks: t.Lacks()}, r.arm(id)
|
||||
}
|
||||
|
||||
// finishArmed finishes a transfer that lacks nothing at an open, its idle bound armed first (H-A-481): a finish cut
|
||||
// by a restart leaves a kept transfer its key still drops, and the next open finishes it. A key not written fails the
|
||||
// open loudly before the finish, the transfer kept for the next open.
|
||||
func (r *Receiver[M]) finishArmed(t Transfer[M]) (Received, error) {
|
||||
if err := r.arm(t.ID); err != nil {
|
||||
return Received{}, err
|
||||
}
|
||||
return Received{ID: t.ID, Ended: true}, r.finish(t)
|
||||
}
|
||||
|
||||
// arm bounds an open transfer by its idle key. A key that cannot be written fails the open loudly (the transfer is
|
||||
// kept, so the opener's next open of it answers what is held and arms it again).
|
||||
func (r *Receiver[M]) arm(id string) error {
|
||||
|
||||
@@ -289,6 +289,78 @@ func aHolderTakesAFileInParts(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
// Design 47 H2 H-A-481: a transfer is finished or taken over, never left open. A finish cut by a restart leaves the
|
||||
// transfer kept with every part held and its chunks in the objects (cut): the opener's next open, or another caller's
|
||||
// with the same digest, finishes it; a transfer that lacks a part is another caller's once its last part is older
|
||||
// than TakeOverAfter, refused to it before; and a transfer that lacks nothing at its open is armed before its finish.
|
||||
t.Run("a transfer is finished or taken over, never left open", func(t *testing.T) {
|
||||
var order []string
|
||||
pkg.Arm = func(id string, _ time.Duration) error { order = append(order, "arm "+id); return nil }
|
||||
store := pkg.Store
|
||||
defer func() { pkg.Arm, pkg.Holds, pkg.Store = nil, nil, store }()
|
||||
cut := func(id string, lacked bool) {
|
||||
tr := Transfer[packageShape]{ID: id, Opener: "builder", Parts: []Want{part}, Chunks: map[string]int{part.Digest: 1},
|
||||
Held: []string{part.Digest}, Opened: now, Last: now, Shape: packageShape{"m", "1"}}
|
||||
if lacked {
|
||||
tr.Held, tr.Chunks = nil, map[string]int{part.Digest: 1}
|
||||
objects[pkg.chunkName(id, part.Digest, 0)] = blob[:1000]
|
||||
} else {
|
||||
objects[pkg.chunkName(id, part.Digest, 0)] = blob
|
||||
}
|
||||
if err := pkg.Transfers.Put(tr); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name, opener string
|
||||
lacked bool
|
||||
after time.Duration
|
||||
want string // "stored", "lacks" (taken over or gone on) or the refusal's words
|
||||
}{
|
||||
{"the opener opens a cut finish again", "builder", false, 0, "stored"},
|
||||
{"another caller opens a cut finish", "other", false, 0, "stored"},
|
||||
{"another caller while the opener sends", "other", true, time.Minute, "another caller, whose last part came 1m0s ago"},
|
||||
{"another caller once the last part is two minutes old", "other", true, TakeOverAfter, "lacks"},
|
||||
} {
|
||||
pkgLog = nil
|
||||
cut("c1", c.lacked)
|
||||
now = now.Add(c.after)
|
||||
got, err := pkg.Open("c1", c.opener, []Want{part}, packageShape{"m", "1"})
|
||||
kept, open, _ := pkg.Transfers.Get("c1")
|
||||
switch c.want {
|
||||
case "stored":
|
||||
if err != nil || !got.Ended || open || len(pkgLog) != 1 || pkgLog[0].outcome != "stored" || string(pkgLog[0].parts[part.Digest]) != string(blob) {
|
||||
t.Errorf("%s: open = %+v, %v, still open %v, log %+v; want it finished and stored", c.name, got, err, open, pkgLog)
|
||||
}
|
||||
case "lacks":
|
||||
if err != nil || got.Ended || !open || kept.Opener != c.opener || !reflect.DeepEqual(got.Lacks, []string{part.Digest}) {
|
||||
t.Errorf("%s: open = %+v, %v, opener %q; want it taken over by %s, lacking the part", c.name, got, err, kept.Opener, c.opener)
|
||||
}
|
||||
if r, err := pkg.Take("c1", c.opener, part.Digest, 1, true, blob[1000:], nil); err != nil || !r.Ended {
|
||||
t.Errorf("%s: the new opener's last chunk = %+v, %v", c.name, r, err)
|
||||
}
|
||||
default:
|
||||
if err == nil || !strings.Contains(err.Error(), c.want) || !open || kept.Opener != "builder" {
|
||||
t.Errorf("%s: open = %v, opener %q; want refused with %q, the transfer kept", c.name, err, kept.Opener, c.want)
|
||||
}
|
||||
_, _ = pkg.Drop("c1")
|
||||
}
|
||||
if len(objects) != 0 {
|
||||
t.Errorf("%s: %d objects left", c.name, len(objects))
|
||||
}
|
||||
}
|
||||
order = nil
|
||||
pkg.Holds = func(Transfer[packageShape], string) bool { return true }
|
||||
pkg.Store = func(tr Transfer[packageShape], _ map[string][]byte) error {
|
||||
order = append(order, "store "+tr.ID)
|
||||
return nil
|
||||
}
|
||||
if got, err := pkg.Open("c2", "builder", []Want{part}, packageShape{"m", "1"}); err != nil || !got.Ended ||
|
||||
!reflect.DeepEqual(order, []string{"arm c2", "store c2"}) {
|
||||
t.Errorf("a transfer lacking nothing at its open = %+v, %v, %v; want its idle bound armed before its finish", got, err, order)
|
||||
}
|
||||
})
|
||||
|
||||
// An open transfer is bounded by its idle key (Arm); the key's end (Expired) drops it idle, arms it again for the
|
||||
// rest when it took a chunk since, and leaves a gone one; a key not written fails the open loudly. The cap the
|
||||
// binding carries (Bound) counts over Cap.
|
||||
|
||||
Reference in New Issue
Block a user