Files
mesh-delivery/sdk/parts/parts.go
T
jschoubben 541eaa20fe
mesh/delivery merged: source.main moved to novox/mesh-delivery@21eaa55b277a
mesh/delivery-group group fix/47-G-bridge merged: every member is merged
mesh/check checking 541eaa20fec4 on 21eaa55b277a
Bridge for G's take-in: the pre-G package directories contracts/go and sdk again, unused
The running pre-G indexer publishes a repository's packages only when every package directory it published last is
still in the tree; G moved contracts/go and sdk to modules/, so mesh-delivery's packages, G's renamed paths among them,
are never published and nothing importing them builds (2026-11-13, indexer journal 'its packages are not published').
These are the trees of e70754ef, unchanged, without sdk/module.json (no second module). Nothing imports them; they go
once G's indexer runs and publishes without them.

Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
2026-11-13 14:34:40 +01:00

146 lines
4.7 KiB
Go

// Package parts is the runtime's sender of a file in parts (novox/hq ADR 0611, ADR 0627 §2, design 51 *Publishing*):
// the caller opens a transfer naming every part's digest and size, the receiver answers the parts it lacks, and only
// those travel, in chunks a bus message carries, in sequence with the last marked. A cut transfer goes on from what
// the receiver answers when the same transfer is opened again. It names no seat, no verb and no seat's type: the
// caller's Open and Chunk are its own calls (a registry seat's client binds them to that seat's verbs and types), so
// this package holds no transport and no contract.
package parts
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
)
// MaxChunk is the most bytes of a part one chunk carries: base64 makes it about 1 000 KiB, under 1 MiB, the server's
// largest message, with the message's own fields (design 51: each part travels in chunks of at most 1 MiB).
const MaxChunk = 760 << 10
// Part is one part of a transfer: its digest (sha256:<hex>), its size and how to read its bytes.
type Part struct {
Digest string
Size int64
Open func() (io.ReadCloser, error)
}
// Opened is the receiver's answer to opening a transfer: whether it holds the whole already, and the digests of the
// parts it lacks.
type Opened struct {
Held bool
Lacks []string
}
// Open opens (or opens again) the transfer on the receiver, naming every part: the caller's own call.
type Open func(ctx context.Context, parts []Part) (Opened, error)
// Chunk sends one chunk of a part: the part's digest, its sequence from 0, whether it is the part's last, its bytes.
type Chunk func(ctx context.Context, part string, sequence int, last bool, data []byte) error
// Sent is what a Send did: whether the receiver held the whole, the parts it lacked and were sent, the chunks they
// took.
type Sent struct {
Held bool
Parts []string
Chunks int
}
// Send opens the transfer and sends the parts the receiver answers it lacks, each in chunks of at most MaxChunk, in
// sequence, the last marked. A part whose bytes do not hash to its digest is refused before anything of it is sent; a
// part the receiver does not name is never read; a part the receiver names that the transfer does not is refused.
func Send(ctx context.Context, open Open, chunk Chunk, parts []Part) (Sent, error) {
byDigest := map[string]Part{}
for _, p := range parts {
byDigest[p.Digest] = p
}
opened, err := open(ctx, parts)
if err != nil {
return Sent{}, err
}
sent := Sent{Held: opened.Held}
for _, d := range opened.Lacks {
p, ok := byDigest[d]
if !ok {
return sent, fmt.Errorf("the receiver lacks the part %s, which the transfer does not name", d)
}
if err := ctx.Err(); err != nil {
return sent, err
}
if err := check(p); err != nil {
return sent, err
}
n, err := sendPart(ctx, chunk, p)
sent.Chunks += n
if err != nil {
return sent, err
}
sent.Parts = append(sent.Parts, d)
}
return sent, nil
}
// check reads the part once and refuses it when its bytes are not its digest or its size.
func check(p Part) error {
r, err := p.Open()
if err != nil {
return fmt.Errorf("the part %s cannot be read: %w", p.Digest, err)
}
defer r.Close()
h := sha256.New()
n, err := io.Copy(h, r)
if err != nil {
return fmt.Errorf("the part %s cannot be read: %w", p.Digest, err)
}
if got := "sha256:" + hex.EncodeToString(h.Sum(nil)); got != p.Digest || n != p.Size {
return fmt.Errorf("the part %s holds %d bytes of digest %s, not its own %d: nothing of it is sent", p.Digest, n, got, p.Size)
}
return nil
}
func sendPart(ctx context.Context, chunk Chunk, p Part) (int, error) {
r, err := p.Open()
if err != nil {
return 0, fmt.Errorf("the part %s cannot be read: %w", p.Digest, err)
}
defer r.Close()
buf := make([]byte, MaxChunk)
// A chunk is sent once the next read says whether it is the last, so the last is always marked: an empty part is
// one empty chunk, marked last.
var prev []byte
sent := 0
send := func(data []byte, last bool) error {
if err := chunk(ctx, p.Digest, sent, last, data); err != nil {
return fmt.Errorf("chunk %d of the part %s: %w", sent, p.Digest, err)
}
sent++
return nil
}
for {
if err := ctx.Err(); err != nil {
return sent, err
}
n, err := io.ReadFull(r, buf)
end := errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF)
if err != nil && !end {
return sent, fmt.Errorf("the part %s cannot be read: %w", p.Digest, err)
}
if end && n == 0 {
if prev == nil {
prev = []byte{}
}
return sent, send(prev, true)
}
if prev != nil {
if err := send(prev, false); err != nil {
return sent, err
}
}
prev = append([]byte(nil), buf[:n]...)
if end {
return sent, send(prev, true)
}
}
}