diff --git a/go/packages_list_test.go b/go/packages_list_test.go index 857ca19..165ca36 100644 --- a/go/packages_list_test.go +++ b/go/packages_list_test.go @@ -25,7 +25,7 @@ import ( var meshSDKPackages = []string{ ".", "announce", "announcetest", "buildstep", "buscalls", "buscompose", "busstate", "busretry", "calladdress", "catalogue", "channel", "declaration", "delivery", "forgekey", "handacts", "health", "internal/beside", "keys", "kvlimit", "membership", "motion", - "objectlink", "outward", "overlay", "parts", "pernode", "process", "provider", "provider/harness", "provisions", + "objectlink", "outward", "overlay", "pernode", "process", "provider", "provider/harness", "provisions", "sdkversion", "seal", "seatdef", "seatlist", "seatlist/seatlisttest", "testbus", "views", "views/viewstest", } diff --git a/go/parts/bus.go b/go/parts/bus.go deleted file mode 100644 index 6dc9d4d..0000000 --- a/go/parts/bus.go +++ /dev/null @@ -1,154 +0,0 @@ -package parts - -import ( - "encoding/json" - "errors" - "fmt" - "sync/atomic" - "time" - - stdio "git.novox.be/novox/mesh-sdk/go" - "git.novox.be/novox/mesh-sdk/go/busstate" -) - -// BusObjects is a Receiver's Objects over the holder's own object store, asked for in `requires` by local (the -// template objects, novox/hq ADR 0595 §2), through the runtime: every holder binds the same adapter, none its own. -func BusObjects(local string) Objects { return busObjects{busstate.ObjectsOf(local)} } - -type busObjects struct{ o busstate.Objects } - -func (b busObjects) Put(name string, data []byte) error { - _, err := b.o.Put(name, data, "") - return err -} -func (b busObjects) Get(name string) ([]byte, error) { - d, err := b.o.Get(name) - return d.Data, err -} -func (b busObjects) Delete(name string) error { return b.o.Delete(name) } -func (b busObjects) Used() (uint64, error) { - f, err := b.o.Fill() - return f.Bytes, err -} - -// BusTransfers is a Receiver's Transfers over the holder's own key-value state of that name (the template state), -// through the runtime, each transfer under its id. -func BusTransfers[M any](state string) Transfers[M] { return busTransfers[M]{state} } - -type busTransfers[M any] struct{ state string } - -func (b busTransfers[M]) Get(id string) (Transfer[M], bool, error) { - var t Transfer[M] - e, err := stdio.State(b.state).Get(id) - if err != nil || e == nil { - return t, false, err - } - return t, true, json.Unmarshal(e.Value, &t) -} -func (b busTransfers[M]) Put(t Transfer[M]) error { - if t.ID == "" { - return errors.New("a transfer without an id cannot be kept") - } - _, err := stdio.State(b.state).Put(t.ID, t) - return err -} -func (b busTransfers[M]) Delete(id string) error { return stdio.State(b.state).Delete(id) } -func (b busTransfers[M]) List() ([]Transfer[M], error) { - keys, err := stdio.State(b.state).Keys() - if err != nil { - return nil, err - } - var out []Transfer[M] - for _, k := range keys { - if t, ok, err := b.Get(k); err == nil && ok { - out = append(out, t) - } - } - return out, nil -} - -// BusCap is a Receiver's Bound over the holder's own object store of that name: the cap its binding carries (the bus's -// own bound, the store's max bytes), read through the runtime once it answers one, since a binding's cap does not -// change while the bundle runs. Make it once per bundle, not per receiver. -func BusCap(local string) func() (uint64, error) { - var known atomic.Uint64 - return func() (uint64, error) { - if c := known.Load(); c > 0 { - return c, nil - } - f, err := busstate.ObjectsOf(local).Fill() - if err != nil { - return 0, fmt.Errorf("%s's cap cannot be read from its binding: %w", local, err) - } - if f.MaxBytes == 0 { - return 0, fmt.Errorf("%s's binding carries no cap", local) - } - known.Store(f.MaxBytes) - return f.MaxBytes, nil - } -} - -// BusArm is a Receiver's Arm over the holder's own key-value state of that name, declared key-ttl: one key per open -// transfer with its own time to live; a key already there is the same bound, kept. -func BusArm(state string) func(id string, ttl time.Duration) error { - return func(id string, ttl time.Duration) error { - if _, err := stdio.State(state).CreateTTL(id, map[string]any{"transfer": id}, ttl); err != nil && !errors.Is(err, stdio.ErrChanged) { - return err - } - return nil - } -} - -// FollowIdle follows the holder's idle state (the one BusArm writes) once the bundle serves, and hands the end of each -// key (the bus's marker, OpExpire) to expired, a Receiver's Expired: the event drops an idle transfer, no timer runs. -// Once the watch is taken it hands every transfer open (open, such as IDs of the holder's Transfers) to expired too, so -// a key that ended while the bundle was down, whose marker no watch saw, still drops its transfer, and a key lost is -// armed again. said is told each outcome (dropped, or the error that stopped it) for the holder's log. It returns -// once the start's look is done; run it in its own goroutine. -func FollowIdle(state string, open func() ([]string, error), expired func(id string) (bool, error), said func(id string, dropped bool, err error)) { - <-stdio.Served() - stdio.State(state).Follow("", func(ch stdio.StateChange) error { - if ch.Op != stdio.OpExpire { - return nil - } - dropped, err := expired(ch.Key) - if said != nil && (dropped || err != nil) { - said(ch.Key, dropped, err) - } - return err - }) - expireOpen(state, open, expired, said) -} - -// expireOpen is FollowIdle's look at its start: every open transfer handed to expired, as its key's end would. -func expireOpen(state string, open func() ([]string, error), expired func(id string) (bool, error), said func(id string, dropped bool, err error)) { - say := func(id string, dropped bool, err error) { - if said != nil && (dropped || err != nil) { - said(id, dropped, err) - } - } - ids, err := open() - if err != nil { - say("", false, fmt.Errorf("the open transfers cannot be read at the start of %s's watch, so one whose idle key ended while the bundle was down waits for a look by hand: %w", state, err)) - return - } - for _, id := range ids { - dropped, err := expired(id) - say(id, dropped, err) - } -} - -// IDs answers the ids of the transfers open in t: FollowIdle's open. -func IDs[M any](t Transfers[M]) func() ([]string, error) { - return func() ([]string, error) { - all, err := t.List() - if err != nil { - return nil, err - } - out := make([]string, 0, len(all)) - for _, x := range all { - out = append(out, x.ID) - } - return out, nil - } -} diff --git a/go/parts/parts.go b/go/parts/parts.go deleted file mode 100644 index e16c463..0000000 --- a/go/parts/parts.go +++ /dev/null @@ -1,145 +0,0 @@ -// Package parts is the runtime's sender of a file in parts (novox/hq ADR 0611, ADR 0627 §2, design 51 *Publishing*): -// the caller opens a transfer naming every part's digest and size, the receiver answers the parts it lacks, and only -// those travel, in chunks a bus message carries, in sequence with the last marked. A cut transfer goes on from what -// the receiver answers when the same transfer is opened again. It names no seat, no verb and no seat's type: the -// caller's Open and Chunk are its own calls (a registry seat's client binds them to that seat's verbs and types), so -// this package holds no transport and no contract. -package parts - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "errors" - "fmt" - "io" -) - -// MaxChunk is the most bytes of a part one chunk carries: base64 makes it about 1 000 KiB, under 1 MiB, the server's -// largest message, with the message's own fields (design 51: each part travels in chunks of at most 1 MiB). -const MaxChunk = 760 << 10 - -// Part is one part of a transfer: its digest (sha256:), its size and how to read its bytes. -type Part struct { - Digest string - Size int64 - Open func() (io.ReadCloser, error) -} - -// Opened is the receiver's answer to opening a transfer: whether it holds the whole already, and the digests of the -// parts it lacks. -type Opened struct { - Held bool - Lacks []string -} - -// Open opens (or opens again) the transfer on the receiver, naming every part: the caller's own call. -type Open func(ctx context.Context, parts []Part) (Opened, error) - -// Chunk sends one chunk of a part: the part's digest, its sequence from 0, whether it is the part's last, its bytes. -type Chunk func(ctx context.Context, part string, sequence int, last bool, data []byte) error - -// Sent is what a Send did: whether the receiver held the whole, the parts it lacked and were sent, the chunks they -// took. -type Sent struct { - Held bool - Parts []string - Chunks int -} - -// Send opens the transfer and sends the parts the receiver answers it lacks, each in chunks of at most MaxChunk, in -// sequence, the last marked. A part whose bytes do not hash to its digest is refused before anything of it is sent; a -// part the receiver does not name is never read; a part the receiver names that the transfer does not is refused. -func Send(ctx context.Context, open Open, chunk Chunk, parts []Part) (Sent, error) { - byDigest := map[string]Part{} - for _, p := range parts { - byDigest[p.Digest] = p - } - opened, err := open(ctx, parts) - if err != nil { - return Sent{}, err - } - sent := Sent{Held: opened.Held} - for _, d := range opened.Lacks { - p, ok := byDigest[d] - if !ok { - return sent, fmt.Errorf("the receiver lacks the part %s, which the transfer does not name", d) - } - if err := ctx.Err(); err != nil { - return sent, err - } - if err := check(p); err != nil { - return sent, err - } - n, err := sendPart(ctx, chunk, p) - sent.Chunks += n - if err != nil { - return sent, err - } - sent.Parts = append(sent.Parts, d) - } - return sent, nil -} - -// check reads the part once and refuses it when its bytes are not its digest or its size. -func check(p Part) error { - r, err := p.Open() - if err != nil { - return fmt.Errorf("the part %s cannot be read: %w", p.Digest, err) - } - defer r.Close() - h := sha256.New() - n, err := io.Copy(h, r) - if err != nil { - return fmt.Errorf("the part %s cannot be read: %w", p.Digest, err) - } - if got := "sha256:" + hex.EncodeToString(h.Sum(nil)); got != p.Digest || n != p.Size { - return fmt.Errorf("the part %s holds %d bytes of digest %s, not its own %d: nothing of it is sent", p.Digest, n, got, p.Size) - } - return nil -} - -func sendPart(ctx context.Context, chunk Chunk, p Part) (int, error) { - r, err := p.Open() - if err != nil { - return 0, fmt.Errorf("the part %s cannot be read: %w", p.Digest, err) - } - defer r.Close() - buf := make([]byte, MaxChunk) - // A chunk is sent once the next read says whether it is the last, so the last is always marked: an empty part is - // one empty chunk, marked last. - var prev []byte - sent := 0 - send := func(data []byte, last bool) error { - if err := chunk(ctx, p.Digest, sent, last, data); err != nil { - return fmt.Errorf("chunk %d of the part %s: %w", sent, p.Digest, err) - } - sent++ - return nil - } - for { - if err := ctx.Err(); err != nil { - return sent, err - } - n, err := io.ReadFull(r, buf) - end := errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) - if err != nil && !end { - return sent, fmt.Errorf("the part %s cannot be read: %w", p.Digest, err) - } - if end && n == 0 { - if prev == nil { - prev = []byte{} - } - return sent, send(prev, true) - } - if prev != nil { - if err := send(prev, false); err != nil { - return sent, err - } - } - prev = append([]byte(nil), buf[:n]...) - if end { - return sent, send(prev, true) - } - } -} diff --git a/go/parts/parts_test.go b/go/parts/parts_test.go deleted file mode 100644 index b3ecf06..0000000 --- a/go/parts/parts_test.go +++ /dev/null @@ -1,122 +0,0 @@ -package parts - -import ( - "bytes" - "context" - "crypto/sha256" - "encoding/hex" - "io" - "strings" - "testing" -) - -// A file travels in parts (ADR 0611, ADR 0627 §2, design 51 *Publishing*; SDK-1), both sides in one test, since -// mesh-sdk is over its test budget: the sender's and the holder's (aHolderTakesAFileInParts, receive_test.go). -func TestAFileTravelsInTheParts(t *testing.T) { - t.Run("the sender sends only what the holder lacks", theSenderSendsOnlyWhatTheHolderLacks) - t.Run("a holder takes a file in parts", aHolderTakesAFileInParts) -} - -// Only the parts the holder says it lacks are read and sent, each in chunks of at most MaxChunk in sequence with the -// last marked, so a cut transfer opened again goes on from what the holder holds; a part whose bytes are not its -// digest is refused before any of it travels. -func theSenderSendsOnlyWhatTheHolderLacks(t *testing.T) { - big := bytes.Repeat([]byte("x"), MaxChunk+10) - small, empty := []byte("config"), []byte{} - part := func(b []byte, opened *int) Part { - sum := sha256.Sum256(b) - return Part{Digest: "sha256:" + hex.EncodeToString(sum[:]), Size: int64(len(b)), - Open: func() (io.ReadCloser, error) { *opened++; return io.NopCloser(bytes.NewReader(b)), nil }} - } - type chunk struct { - part string - seq int64 - last bool - data []byte - } - type receiver struct { - open Open - chunk Chunk - } - holder := func(lacks []string, got *[]chunk, asked *[]string) receiver { - return receiver{ - open: func(_ context.Context, ps []Part) (Opened, error) { - *asked = append(*asked, "open") - return Opened{Lacks: lacks, Held: len(lacks) == 0}, nil - }, - chunk: func(_ context.Context, part string, seq int, last bool, data []byte) error { - if len(data) > MaxChunk { - t.Errorf("a chunk is %d bytes, over MaxChunk", len(data)) - } - *got = append(*got, chunk{part, int64(seq), last, append([]byte(nil), data...)}) - return nil - }, - } - } - var opens [3]int - parts := []Part{part(big, &opens[0]), part(small, &opens[1]), part(empty, &opens[2])} - - for _, c := range []struct { - name string - lacks []int - chunks int - err string - }{ - {"a fresh transfer sends every part, the big one in two chunks and the empty one as one", []int{0, 1, 2}, 4, ""}, - {"a cut transfer opened again sends only what the holder lacks", []int{2}, 1, ""}, - {"a digest the holder holds sends nothing", nil, 0, ""}, - } { - opens = [3]int{} - var lacks []string - for _, i := range c.lacks { - lacks = append(lacks, parts[i].Digest) - } - var got []chunk - var asked []string - r := holder(lacks, &got, &asked) - sent, err := Send(context.Background(), r.open, r.chunk, parts) - if err != nil || sent.Chunks != c.chunks || len(got) != c.chunks || len(sent.Parts) != len(c.lacks) { - t.Errorf("%s: sent %+v, %d chunks heard, %v", c.name, sent, len(got), err) - continue - } - for i := range parts { - want := 0 - for _, l := range c.lacks { - if l == i { - want = 2 // read once to check its digest, once to send it - } - } - if opens[i] != want { - t.Errorf("%s: part %d read %d times, want %d", c.name, i, opens[i], want) - } - } - whole := map[string][]byte{} - for i, ch := range got { - whole[ch.part] = append(whole[ch.part], ch.data...) - lastOfPart := i == len(got)-1 || got[i+1].part != ch.part - if ch.last != lastOfPart || (i > 0 && got[i-1].part == ch.part && got[i-1].seq+1 != ch.seq) { - t.Errorf("%s: chunk %d of %s is %d, last %v", c.name, i, ch.part[:12], ch.seq, ch.last) - } - } - for _, l := range c.lacks { - if p := parts[l]; int64(len(whole[p.Digest])) != p.Size { - t.Errorf("%s: part %d arrived as %d bytes of %d", c.name, l, len(whole[p.Digest]), p.Size) - } - } - } - - // A part whose bytes are not its digest travels not at all; a part the holder names that the transfer does not is - // refused, naming it. - liar := part(small, &opens[1]) - liar.Digest = parts[0].Digest - var got []chunk - var asked []string - r := holder([]string{liar.Digest}, &got, &asked) - if _, err := Send(context.Background(), r.open, r.chunk, []Part{liar}); err == nil || len(got) != 0 || !strings.Contains(err.Error(), "nothing of it is sent") { - t.Errorf("a part not its digest: %v, %d chunks sent", err, len(got)) - } - r = holder([]string{"sha256:" + strings.Repeat("f", 64)}, &got, &asked) - if _, err := Send(context.Background(), r.open, r.chunk, parts); err == nil || !strings.Contains(err.Error(), "does not name") { - t.Errorf("a part the transfer does not name: %v", err) - } -} diff --git a/go/parts/receive.go b/go/parts/receive.go deleted file mode 100644 index c77bcfe..0000000 --- a/go/parts/receive.go +++ /dev/null @@ -1,441 +0,0 @@ -package parts - -// The receiving side of a file in parts (novox/hq ADR 0611, ADR 0627 §2, design 51 *Publishing*): a holder opens a -// transfer naming every part, takes the chunks of the parts it lacks into its own objects, checks each whole part's -// hash, drops a transfer left idle when its idle bound ends and says when its objects fill. Like the sender it names no seat, no verb and no -// seat's type: the caller's own shape of an opening rides along as M, its objects and its state are its own, and the -// checks before storing, the storing and the saying of a refusal are the caller's functions. Each holder binds it to -// the seat it holds. - -import ( - "bytes" - "crypto/sha256" - "encoding/hex" - "errors" - "fmt" - "slices" - "sort" - "strings" - "time" -) - -// The bounds design 51 gives the receiving side; a Receiver's own fields override them. -const ( - // MaxReceived is the most bytes one chunk may carry when it arrives. - MaxReceived = 1 << 20 - // IdleAfter is how long a transfer with no chunk stays open. - IdleAfter = 24 * time.Hour - // FullPercent is the fill of the objects past which the receiver says they fill. - FullPercent = 80 -) - -// Want is one part an opening names: its digest (sha256:) and its size. -type Want struct { - Digest string `json:"digest"` - Size int64 `json:"size"` -} - -// Transfer is one open transfer, kept in the caller's state; its chunks are objects of the caller's object store. Shape -// is the caller's own body of the opening, kept as it came. -type Transfer[M any] struct { - ID string `json:"id"` - Opener string `json:"opener"` - Parts []Want `json:"parts"` - // Held are the parts whose every chunk arrived and hashed right, or that the holder held at the open; Chunks the - // chunks taken per part. - Held []string `json:"held,omitempty"` - Chunks map[string]int `json:"chunks,omitempty"` - Opened time.Time `json:"opened"` - Last time.Time `json:"last"` - Shape M `json:"shape"` -} - -// Lacks is the digests of the parts the transfer does not hold yet, sorted. -func (t Transfer[M]) Lacks() []string { - var out []string - for _, p := range t.Parts { - if !slices.Contains(t.Held, p.Digest) && !slices.Contains(out, p.Digest) { - out = append(out, p.Digest) - } - } - sort.Strings(out) - return out -} - -// Taken says whether any chunk of the part was taken in this transfer (a part held at the open was not). -func (t Transfer[M]) Taken(digest string) bool { - _, ok := t.Chunks[digest] - return ok -} - -// Objects is the holder's own object store the chunks go into. -type Objects interface { - Put(name string, data []byte) error - Get(name string) ([]byte, error) - Delete(name string) error - // Used is the bytes the store holds. - Used() (uint64, error) -} - -// Transfers is the holder's own state of open transfers, keyed by Transfer.ID. -type Transfers[M any] interface { - Get(id string) (Transfer[M], bool, error) - Put(t Transfer[M]) error - Delete(id string) error - List() ([]Transfer[M], error) -} - -// Received is the receiver's answer to an open or a chunk: the parts still lacked, and whether the transfer ended -// (stored or refused). -type Received struct { - ID string - Lacks []string - Ended bool -} - -// AtCap opens the reason of a refusal for want of room: the objects would pass their cap. A holder tells it from -// the other refusals with RefusedAtCap, so the finding it raises for it can clear once it takes a file again. -const AtCap = "the objects are at their cap" - -// RefusedAtCap says whether a refusal's reason is one for want of room (AtCap). -func RefusedAtCap(reason string) bool { return strings.HasPrefix(reason, AtCap) } - -// Receiver takes transfers in parts into a holder's own objects. -type Receiver[M any] struct { - Objects Objects - Transfers Transfers[M] - // Prefix starts the name of every chunk object, so two receivers can share one object store. - Prefix string - // Cap is the bytes the objects may hold; an opening that would pass it is refused. Required: it is said where - // Bound answers no cap. - Cap uint64 - // Bound reads the cap the objects' binding carries (BusCap): the bus's own bound, which then counts over Cap. - // Optional; an error or a zero falls back to Cap. - Bound func() (uint64, error) - // MaxChunk, Idle and Full override MaxReceived, IdleAfter and FullPercent when set. - MaxChunk int - Idle time.Duration - Full uint64 - // Holds says the holder already holds a part, which then counts as held at the open and is not sent. Optional. - Holds func(t Transfer[M], digest string) bool - // Check is the caller's check of a whole transfer before anything is stored; its error is the refusal's reason. - // Optional. - Check func(t Transfer[M]) error - // Store stores a whole, checked transfer: the parts taken in it by digest (not those held at the open). It says - // the outcome itself; its error is answered as it is. - Store func(t Transfer[M], parts map[string][]byte) error - // Refuse says a refusal by the receiver (the cap, the caller's check, a part it cannot read back). Its own error is - // joined to the refusal's. - Refuse func(t Transfer[M], reason string) error - // Filled is told the objects' fill after every change: used, the cap and whether it is past Full percent. Optional. - Filled func(used, cap uint64, full bool) - // Arm writes a transfer's idle key with its own time to live, once (BusArm over a state declared key-ttl); one - // already there is no fault. The key's end is the event that calls Expired, so an open transfer left with no - // chunk is dropped without a timer. Optional: nil arms nothing, and only DropIdle drops. - Arm func(id string, ttl time.Duration) error - Now func() time.Time -} - -func (r *Receiver[M]) now() time.Time { - if r.Now != nil { - return r.Now().UTC() - } - return time.Now().UTC() -} - -func (r *Receiver[M]) maxChunk() int { - if r.MaxChunk > 0 { - return r.MaxChunk - } - return MaxReceived -} - -func (r *Receiver[M]) idle() time.Duration { - if r.Idle > 0 { - return r.Idle - } - return IdleAfter -} - -// CapNow is the cap the receiver counts against: the binding's (Bound) when it answers one, else Cap. -func (r *Receiver[M]) CapNow() uint64 { - if r.Bound != nil { - if c, err := r.Bound(); err == nil && c > 0 { - return c - } - } - return r.Cap -} - -func (r *Receiver[M]) fullPct() uint64 { - if r.Full > 0 { - return r.Full - } - return FullPercent -} - -// Open opens a transfer for opener, or opens it again: a transfer already open answers what it lacks, so a cut transfer -// goes on from there, and only its opener may open it again. A new transfer that would pass the cap is refused; parts -// the holder holds count as held; when nothing is lacked the transfer ends at once. -func (r *Receiver[M]) Open(id, opener string, parts []Want, shape M) (Received, error) { - switch { - case strings.TrimSpace(id) == "" || strings.ContainsAny(id, "/. *>"): - return Received{}, fmt.Errorf("transfer %q is no transfer id", id) - case len(parts) == 0: - return Received{}, errors.New("no parts") - } - for _, p := range parts { - if !strings.HasPrefix(p.Digest, "sha256:") || len(p.Digest) != len("sha256:")+64 { - return Received{}, fmt.Errorf("part digest %q is no sha256 digest", p.Digest) - } - } - if t, found, err := r.Transfers.Get(id); err != nil { - return Received{}, err - } else if found { - if t.Opener != opener { - return Received{}, fmt.Errorf("refused for %s: transfer %s was opened by another caller", opener, id) - } - return Received{ID: id, Lacks: t.Lacks()}, r.arm(id) - } - t := Transfer[M]{ID: id, Opener: opener, Parts: parts, Chunks: map[string]int{}, Opened: r.now(), Last: r.now(), Shape: shape} - used, err := r.Objects.Used() - if err != nil { - return Received{}, fmt.Errorf("the objects' fill cannot be read: %w", err) - } - var size uint64 - for _, p := range parts { - size += uint64(max(p.Size, 0)) - } - if cap := r.CapNow(); used+size > cap { - return Received{ID: id, Ended: true}, r.refuse(t, fmt.Sprintf(AtCap+" of %d bytes (%d held, %d asked)", cap, used, size)) - } - if r.Holds != nil { - for _, p := range parts { - if r.Holds(t, p.Digest) { - t.Held = append(t.Held, p.Digest) - } - } - } - if err := r.Transfers.Put(t); err != nil { - return Received{}, fmt.Errorf("transfer %s cannot be kept: %w", id, err) - } - r.fill() - if len(t.Lacks()) == 0 { - return Received{ID: id, Ended: true}, r.finish(t) - } - return Received{ID: id, Lacks: t.Lacks()}, r.arm(id) -} - -// arm bounds an open transfer by its idle key. A key that cannot be written fails the open loudly (the transfer is -// kept, so the opener's next open of it answers what is held and arms it again). -func (r *Receiver[M]) arm(id string) error { - if r.Arm == nil { - return nil - } - if err := r.Arm(id, r.idle()); err != nil { - return fmt.Errorf("transfer %s is open but its idle bound is not kept, so it would never be dropped: %w", id, err) - } - return nil -} - -// Take takes one chunk of a part from the transfer's opener: chunks of a part arrive in sequence from 0, the last -// marked; the whole part must hash to its digest, else its chunks are dropped and the part is lacked again. Belongs, -// when given, refuses a transfer the caller's own shape says is not this call's (another seat's, say). When the last -// lacked part arrives the transfer ends: the caller's check, then its store. -func (r *Receiver[M]) Take(id, opener, digest string, sequence int, last bool, data []byte, belongs func(Transfer[M]) error) (Received, error) { - t, found, err := r.Transfers.Get(id) - switch { - case err != nil: - return Received{}, err - case !found: - return Received{}, fmt.Errorf("transfer %s is not open (dropped after %s with no chunk, or never opened)", id, r.idle()) - case t.Opener != opener: - return Received{}, fmt.Errorf("refused for %s: transfer %s was opened by another caller", opener, id) - } - if belongs != nil { - if err := belongs(t); err != nil { - return Received{}, err - } - } - switch { - case len(data) > r.maxChunk(): - return Received{}, fmt.Errorf("a chunk of %d bytes is over the %d a chunk may carry", len(data), r.maxChunk()) - case !slices.ContainsFunc(t.Parts, func(p Want) bool { return p.Digest == digest }): - return Received{}, fmt.Errorf("transfer %s names no part %s", id, digest) - case slices.Contains(t.Held, digest): - return Received{ID: id, Lacks: t.Lacks()}, nil - case sequence != t.Chunks[digest]: - return Received{}, fmt.Errorf("part %s of transfer %s waits for chunk %d, not %d", digest, id, t.Chunks[digest], sequence) - } - if err := r.Objects.Put(r.chunkName(id, digest, sequence), data); err != nil { - return Received{}, fmt.Errorf("the objects did not take the chunk: %w", err) - } - // A transfer read back from a key-value state has no Chunks until one was taken: the empty map is kept omitted. - if t.Chunks == nil { - t.Chunks = map[string]int{} - } - t.Chunks[digest] = sequence + 1 - t.Last = r.now() - if last { - whole, err := r.part(t, digest) - if err != nil { - return Received{}, err - } - if got := "sha256:" + sum(whole); got != digest { - r.dropChunks(t, digest) - delete(t.Chunks, digest) - _ = r.Transfers.Put(t) - r.fill() - return Received{}, fmt.Errorf("refused: the bytes of part %s hash to %s; send the part again", digest, got) - } - t.Held = append(t.Held, digest) - } - if err := r.Transfers.Put(t); err != nil { - return Received{}, err - } - r.fill() - if len(t.Lacks()) == 0 { - return Received{ID: id, Ended: true}, r.finish(t) - } - return Received{ID: id, Lacks: t.Lacks()}, nil -} - -// finish checks and stores a whole transfer; its chunks and its key go either way. -func (r *Receiver[M]) finish(t Transfer[M]) error { - defer func() { - for _, p := range t.Parts { - r.dropChunks(t, p.Digest) - } - _ = r.Transfers.Delete(t.ID) - r.fill() - }() - if r.Check != nil { - if err := r.Check(t); err != nil { - return r.refuse(t, err.Error()) - } - } - parts := map[string][]byte{} - for _, p := range t.Parts { - if !t.Taken(p.Digest) { - continue // held by the holder already - } - b, err := r.part(t, p.Digest) - if err != nil { - return r.refuse(t, "part "+p.Digest+" cannot be read from the objects: "+err.Error()) - } - parts[p.Digest] = b - } - return r.Store(t, parts) -} - -func (r *Receiver[M]) refuse(t Transfer[M], reason string) error { - err := errors.New("refused: " + reason) - if r.Refuse != nil { - if serr := r.Refuse(t, reason); serr != nil { - return fmt.Errorf("%w; and the refusal could not be said: %w", err, serr) - } - } - return err -} - -func (r *Receiver[M]) part(t Transfer[M], digest string) ([]byte, error) { - var buf bytes.Buffer - for seq := 0; seq < t.Chunks[digest]; seq++ { - b, err := r.Objects.Get(r.chunkName(t.ID, digest, seq)) - if err != nil { - return nil, err - } - buf.Write(b) - } - return buf.Bytes(), nil -} - -func (r *Receiver[M]) dropChunks(t Transfer[M], digest string) { - for seq := 0; seq < t.Chunks[digest]; seq++ { - _ = r.Objects.Delete(r.chunkName(t.ID, digest, seq)) - } -} - -func (r *Receiver[M]) chunkName(id, digest string, seq int) string { - return fmt.Sprintf("%s%s/%s/%d", r.Prefix, id, strings.TrimPrefix(digest, "sha256:"), seq) -} - -// fill tells Filled the objects' fill. -func (r *Receiver[M]) fill() { - cap := r.CapNow() - if r.Filled == nil || cap == 0 { - return - } - used, err := r.Objects.Used() - if err != nil { - return - } - r.Filled(used, cap, used*100 >= cap*r.fullPct()) -} - -// Drop drops a transfer, by hand or because it was idle: its chunks and its key. -func (r *Receiver[M]) Drop(id string) (Transfer[M], error) { - t, found, err := r.Transfers.Get(id) - if err != nil { - return Transfer[M]{}, err - } - if !found { - return Transfer[M]{}, fmt.Errorf("transfer %s is not open", id) - } - for _, p := range t.Parts { - r.dropChunks(t, p.Digest) - } - if err := r.Transfers.Delete(id); err != nil { - return Transfer[M]{}, err - } - r.fill() - return t, nil -} - -// Expired is told that a transfer's idle key ended (the bus's marker, FollowIdle): a transfer with no chunk for Idle -// is dropped with its chunks; one that took a chunk since is armed again for the rest of its bound; one already gone -// is nothing to do. It answers whether it dropped the transfer. -func (r *Receiver[M]) Expired(id string) (bool, error) { - t, found, err := r.Transfers.Get(id) - if err != nil { - return false, fmt.Errorf("transfer %s's idle bound ended, but it cannot be read: %w", id, err) - } - if !found { - return false, nil - } - idle := r.now().Sub(t.Last) - if idle >= r.idle() { - if _, err := r.Drop(id); err != nil { - return false, fmt.Errorf("transfer %s had no chunk for %s and is not dropped: %w", id, r.idle(), err) - } - return true, nil - } - if r.Arm == nil { - return false, nil - } - return false, r.Arm(id, max(r.idle()-idle, time.Second)) -} - -// DropIdle drops every transfer with no chunk for Idle and answers their ids: a look by hand (a listing tool). The -// drop itself follows each transfer's idle key (Arm, Expired), never a timer. -func (r *Receiver[M]) DropIdle() ([]string, error) { - all, err := r.Transfers.List() - if err != nil { - return nil, err - } - var out []string - for _, t := range all { - if r.now().Sub(t.Last) >= r.idle() { - if _, err := r.Drop(t.ID); err == nil { - out = append(out, t.ID) - } - } - } - sort.Strings(out) - return out, nil -} - -func sum(b []byte) string { - s := sha256.Sum256(b) - return hex.EncodeToString(s[:]) -} diff --git a/go/parts/receive_test.go b/go/parts/receive_test.go deleted file mode 100644 index d4ee5df..0000000 --- a/go/parts/receive_test.go +++ /dev/null @@ -1,305 +0,0 @@ -package parts - -import ( - "crypto/sha256" - "encoding/hex" - "encoding/json" - "errors" - "reflect" - "strings" - "testing" - "time" -) - -type memObjects map[string][]byte - -func (m memObjects) Put(n string, d []byte) error { m[n] = append([]byte(nil), d...); return nil } -func (m memObjects) Get(n string) ([]byte, error) { - if d, ok := m[n]; ok { - return d, nil - } - return nil, errors.New("no object " + n) -} -func (m memObjects) Delete(n string) error { delete(m, n); return nil } -func (m memObjects) Used() (uint64, error) { - var u uint64 - for _, d := range m { - u += uint64(len(d)) - } - return u, nil -} - -// memTransfers keeps each transfer as the bus keeps it, its JSON, so every case reads back what a key-value state -// gives (BusTransfers): an empty map kept with omitempty comes back nil. -type memTransfers[M any] map[string][]byte - -func (m memTransfers[M]) Get(id string) (Transfer[M], bool, error) { - var t Transfer[M] - raw, ok := m[id] - if !ok { - return t, false, nil - } - return t, true, json.Unmarshal(raw, &t) -} -func (m memTransfers[M]) Put(t Transfer[M]) error { - raw, err := json.Marshal(t) - m[t.ID] = raw - return err -} -func (m memTransfers[M]) Delete(id string) error { delete(m, id); return nil } -func (m memTransfers[M]) List() ([]Transfer[M], error) { - var out []Transfer[M] - for id := range m { - t, _, err := m.Get(id) - if err != nil { - return nil, err - } - out = append(out, t) - } - return out, nil -} - -// Two shapes of an opening, as two different seats' holders would keep them. -type packageShape struct{ Name, Version string } -type imageShape struct { - Repository string - Layers int -} - -type ended struct { - id, outcome string - parts map[string][]byte -} - -func digestOf(b []byte) string { s := sha256.Sum256(b); return "sha256:" + hex.EncodeToString(s[:]) } - -func receiverOf[M any](objects memObjects, prefix string, now *time.Time, log *[]ended, check func(Transfer[M]) error) *Receiver[M] { - return &Receiver[M]{Objects: objects, Transfers: memTransfers[M]{}, Prefix: prefix, Cap: 1 << 20, MaxChunk: 1000, - Check: check, Now: func() time.Time { return *now }, - Store: func(t Transfer[M], parts map[string][]byte) error { - *log = append(*log, ended{t.ID, "stored", parts}) - return nil - }, - Refuse: func(t Transfer[M], reason string) error { - *log = append(*log, ended{t.ID, "refused: " + reason, nil}) - return nil - }} -} - -// aHolderTakesAFileInParts (a case of TestAFileTravelsInTheParts) runs the receiving side for two holders with two shapes and two prefixes on one object -// store (ADR 0627 §2): neither names a seat; the shapes, the checks and the saying are each caller's own. -func aHolderTakesAFileInParts(t *testing.T) { - objects := memObjects{} - now := time.Unix(1000, 0) - var pkgLog, imgLog []ended - pkg := receiverOf(objects, "npm/", &now, &pkgLog, func(t Transfer[packageShape]) error { - if t.Shape.Version == "" { - return errors.New("the builder did not number it") - } - return nil - }) - img := receiverOf[imageShape](objects, "image/", &now, &imgLog, nil) - blob := []byte(strings.Repeat("a", 1500)) - part := Want{Digest: digestOf(blob), Size: int64(len(blob))} - layer := []byte("layer") - held := Want{Digest: digestOf(layer), Size: int64(len(layer))} - img.Holds = func(_ Transfer[imageShape], d string) bool { return d == held.Digest } - - t.Run("two holders, one transfer id, two shapes", func(t *testing.T) { - a, err := pkg.Open("t1", "builder", []Want{part}, packageShape{"m/program", "0.1.4"}) - if err != nil || !reflect.DeepEqual(a.Lacks, []string{part.Digest}) { - t.Fatalf("pkg open = %+v, %v", a, err) - } - b, err := img.Open("t1", "builder", []Want{part, held}, imageShape{"m/app", 2}) - if err != nil || !reflect.DeepEqual(b.Lacks, []string{part.Digest}) { - t.Fatalf("img open = %+v, %v: a part the holder holds is not lacked", b, err) - } - for _, r := range []func(int, bool, []byte) (Received, error){ - func(s int, l bool, d []byte) (Received, error) { - return pkg.Take("t1", "builder", part.Digest, s, l, d, nil) - }, - func(s int, l bool, d []byte) (Received, error) { - return img.Take("t1", "builder", part.Digest, s, l, d, nil) - }, - } { - if _, err := r(0, false, blob[:1000]); err != nil { - t.Fatal(err) - } - } - // A cut transfer opened again goes on from what is held; another caller may not. - if a, _ := pkg.Open("t1", "builder", []Want{part}, packageShape{}); !reflect.DeepEqual(a.Lacks, []string{part.Digest}) { - t.Errorf("reopen = %+v", a) - } - if _, err := pkg.Open("t1", "other", []Want{part}, packageShape{}); err == nil || !strings.Contains(err.Error(), "another caller") { - t.Errorf("reopen by another = %v", err) - } - if _, err := pkg.Take("t1", "builder", part.Digest, 0, false, blob[:1000], nil); err == nil || !strings.Contains(err.Error(), "waits for chunk 1") { - t.Errorf("a chunk out of sequence = %v", err) - } - a, err = pkg.Take("t1", "builder", part.Digest, 1, true, blob[1000:], nil) - if err != nil || !a.Ended || len(pkgLog) != 1 || pkgLog[0].outcome != "stored" || string(pkgLog[0].parts[part.Digest]) != string(blob) { - t.Fatalf("pkg end = %+v, %v, %+v", a, err, pkgLog) - } - // The image holder's chunks stood apart under its own prefix and are still there. - if _, ok := objects["image/t1/"+strings.TrimPrefix(part.Digest, "sha256:")+"/0"]; !ok || len(objects) != 1 { - t.Fatalf("objects after pkg's end = %v keys, want only the image holder's chunk", len(objects)) - } - b, err = img.Take("t1", "builder", part.Digest, 1, true, blob[1000:], func(t Transfer[imageShape]) error { return nil }) - if err != nil || !b.Ended || len(imgLog) != 1 || len(imgLog[0].parts) != 1 { - t.Fatalf("img end = %+v, %v, %+v: only the part taken is stored", b, err, imgLog) - } - if len(objects) != 0 { - t.Errorf("objects after both ends = %d, want none", len(objects)) - } - }) - - t.Run("refusals", func(t *testing.T) { - for _, c := range []struct { - name, want string - do func() error - }{ - {"no transfer id", "no transfer id", func() error { _, err := pkg.Open("a/b", "builder", []Want{part}, packageShape{}); return err }}, - {"no sha256 digest", "no sha256 digest", func() error { - _, err := pkg.Open("t", "builder", []Want{{Digest: "md5:x"}}, packageShape{}) - return err - }}, - {"another caller's chunk", "another caller", func() error { - _, _ = pkg.Open("t2", "builder", []Want{part}, packageShape{"m", "1"}) - _, err := pkg.Take("t2", "other", part.Digest, 0, true, blob, nil) - return err - }}, - {"a chunk over the bound", "over the 1000", func() error { - _, err := pkg.Take("t2", "builder", part.Digest, 0, false, make([]byte, 1001), nil) - return err - }}, - {"a part the transfer does not name", "names no part", func() error { - _, err := pkg.Take("t2", "builder", held.Digest, 0, true, layer, nil) - return err - }}, - {"the caller's own belongs", "not this seat's", func() error { - _, err := pkg.Take("t2", "builder", part.Digest, 0, true, blob, func(Transfer[packageShape]) error { return errors.New("not this seat's") }) - return err - }}, - {"bytes that do not hash", "send the part again", func() error { - _, err := pkg.Take("t2", "builder", part.Digest, 0, true, []byte("other"), nil) - return err - }}, - {"the caller's check", "refused: the builder did not number it", func() error { - _, _ = pkg.Open("t3", "builder", []Want{part}, packageShape{Name: "m"}) - _, _ = pkg.Take("t3", "builder", part.Digest, 0, false, blob[:1000], nil) - _, err := pkg.Take("t3", "builder", part.Digest, 1, true, blob[1000:], nil) - return err - }}, - {"the cap", "at their cap", func() error { - pkg.Cap = 100 - defer func() { pkg.Cap = 1 << 20 }() - _, err := pkg.Open("t4", "builder", []Want{part}, packageShape{}) - return err - }}, - } { - if err := c.do(); err == nil || !strings.Contains(err.Error(), c.want) { - t.Errorf("%s: err = %v, want %q", c.name, err, c.want) - } - } - if got := pkgLog[len(pkgLog)-1].outcome; !strings.HasPrefix(got, "refused: the objects are at their cap") { - t.Errorf("the cap's refusal said %q", got) - } - if !RefusedAtCap(strings.TrimPrefix(pkgLog[len(pkgLog)-1].outcome, "refused: ")) || RefusedAtCap(strings.TrimPrefix(pkgLog[len(pkgLog)-2].outcome, "refused: ")) { - t.Errorf("RefusedAtCap tells the cap's refusal from the check's: %q, %q", pkgLog[len(pkgLog)-1].outcome, pkgLog[len(pkgLog)-2].outcome) - } - if got := pkgLog[len(pkgLog)-2].outcome; got != "refused: the builder did not number it" { - t.Errorf("the check's refusal said %q", got) - } - // A part that did not hash is lacked again, its chunk gone. - if a, _ := pkg.Open("t2", "builder", []Want{part}, packageShape{}); !reflect.DeepEqual(a.Lacks, []string{part.Digest}) { - t.Errorf("after a bad hash = %+v", a) - } - }) - - t.Run("idle drop and the fill", func(t *testing.T) { - var fills []bool - pkg.Filled = func(used, cap uint64, full bool) { fills = append(fills, full) } - pkg.Cap, pkg.Full = 1500, 60 - defer func() { pkg.Cap, pkg.Full, pkg.Filled = 1<<20, 0, nil }() - _, _ = pkg.Open("t5", "builder", []Want{part}, packageShape{"m", "1"}) - _, _ = pkg.Take("t5", "builder", part.Digest, 0, false, blob[:1000], nil) - if len(fills) == 0 || !fills[len(fills)-1] { - t.Errorf("fills = %v, want full at 1000 of 1500 bytes (60 percent)", fills) - } - now = now.Add(IdleAfter) - dropped, err := pkg.DropIdle() - if err != nil || !reflect.DeepEqual(dropped, []string{"t2", "t5"}) { - t.Errorf("DropIdle = %v, %v", dropped, err) - } - if fills[len(fills)-1] || len(objects) != 0 { - t.Errorf("after the drop: full %v, %d objects", fills[len(fills)-1], len(objects)) - } - if _, err := pkg.Take("t5", "builder", part.Digest, 1, true, blob[1000:], nil); err == nil || !strings.Contains(err.Error(), "not open") { - t.Errorf("a chunk after the drop = %v", err) - } - }) - - // An open transfer is bounded by its idle key (Arm); the key's end (Expired) drops it idle, arms it again for the - // rest when it took a chunk since, and leaves a gone one; a key not written fails the open loudly. The cap the - // binding carries (Bound) counts over Cap. - t.Run("the idle key and the binding's cap", func(t *testing.T) { - armed := map[string]time.Duration{} - pkg.Arm = func(id string, ttl time.Duration) error { armed[id] = ttl; return nil } - defer func() { pkg.Arm, pkg.Bound = nil, nil }() - if _, err := pkg.Open("t6", "builder", []Want{part}, packageShape{"m", "1"}); err != nil || armed["t6"] != IdleAfter { - t.Fatalf("open = %v, armed %v; want t6's idle key for %s", err, armed, IdleAfter) - } - now = now.Add(time.Hour) - _, _ = pkg.Take("t6", "builder", part.Digest, 0, false, blob[:1000], nil) - now = now.Add(IdleAfter - time.Hour) - if dropped, err := pkg.Expired("t6"); err != nil || dropped || armed["t6"] != time.Hour { - t.Errorf("expired after a chunk = %v, %v, armed %v; want armed again for the hour left", dropped, err, armed["t6"]) - } - now = now.Add(time.Hour) - if dropped, err := pkg.Expired("t6"); err != nil || !dropped || len(objects) != 0 { - t.Errorf("expired idle = %v, %v, %d objects; want dropped with its chunks", dropped, err, len(objects)) - } - if dropped, err := pkg.Expired("t6"); err != nil || dropped { - t.Errorf("expired once gone = %v, %v; want nothing to do", dropped, err) - } - pkg.Arm = func(string, time.Duration) error { return errors.New("bus down") } - if _, err := pkg.Open("t7", "builder", []Want{part}, packageShape{"m", "1"}); err == nil || !strings.Contains(err.Error(), "never be dropped") { - t.Errorf("open with no idle key = %v, want a loud failure", err) - } - pkg.Bound = func() (uint64, error) { return 100, nil } - if _, err := pkg.Open("t8", "builder", []Want{part}, packageShape{}); err == nil || !strings.Contains(err.Error(), "cap of 100 bytes") { - t.Errorf("open past the binding's cap = %v, want refused at 100 bytes", err) - } - if pkg.Bound = func() (uint64, error) { return 0, errors.New("no binding") }; pkg.CapNow() != pkg.Cap { - t.Errorf("CapNow with no binding read = %d, want Cap %d", pkg.CapNow(), pkg.Cap) - } - pkg.Bound = nil - // At the start of the idle watch every open transfer is looked at (expireOpen): one whose key ended while the - // bundle was down is dropped, one still within its bound is armed again for the rest. - pkg.Arm = func(id string, ttl time.Duration) error { armed[id] = ttl; return nil } - for id := range pkg.Transfers.(memTransfers[packageShape]) { - _, _ = pkg.Drop(id) - } - _, _ = pkg.Open("t9", "builder", []Want{part}, packageShape{"m", "1"}) - now = now.Add(IdleAfter + time.Hour) - _, _ = pkg.Open("t10", "builder", []Want{part}, packageShape{"m", "1"}) - now = now.Add(time.Hour) - var dropped []string - expireOpen("idle", IDs(pkg.Transfers), pkg.Expired, func(id string, d bool, err error) { - if err != nil { - t.Errorf("start's look at %s: %v", id, err) - } - if d { - dropped = append(dropped, id) - } - }) - if !reflect.DeepEqual(dropped, []string{"t9"}) || armed["t10"] != IdleAfter-time.Hour { - t.Errorf("at the start: dropped %v, t10 armed for %s; want t9 dropped and t10 armed for %s", dropped, armed["t10"], IdleAfter-time.Hour) - } - var said error - expireOpen("idle", func() ([]string, error) { return nil, errors.New("bus down") }, pkg.Expired, func(_ string, _ bool, err error) { said = err }) - if said == nil || !strings.Contains(said.Error(), "bus down") { - t.Errorf("start's look with no list said %v, want the failure", said) - } - }) -}