The running pre-G indexer publishes a repository's packages only when every package directory it published last is
still in the tree; G moved contracts/go and sdk to modules/, so mesh-delivery's packages, G's renamed paths among them,
are never published and nothing importing them builds (2026-11-13, indexer journal 'its packages are not published').
These are the trees of e70754ef, unchanged, without sdk/module.json (no second module). Nothing imports them; they go
once G's indexer runs and publishes without them.
Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
146 lines
4.7 KiB
Go
146 lines
4.7 KiB
Go
// Package parts is the runtime's sender of a file in parts (novox/hq ADR 0611, ADR 0627 §2, design 51 *Publishing*):
|
|
// the caller opens a transfer naming every part's digest and size, the receiver answers the parts it lacks, and only
|
|
// those travel, in chunks a bus message carries, in sequence with the last marked. A cut transfer goes on from what
|
|
// the receiver answers when the same transfer is opened again. It names no seat, no verb and no seat's type: the
|
|
// caller's Open and Chunk are its own calls (a registry seat's client binds them to that seat's verbs and types), so
|
|
// this package holds no transport and no contract.
|
|
package parts
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
)
|
|
|
|
// MaxChunk is the most bytes of a part one chunk carries: base64 makes it about 1 000 KiB, under 1 MiB, the server's
|
|
// largest message, with the message's own fields (design 51: each part travels in chunks of at most 1 MiB).
|
|
const MaxChunk = 760 << 10
|
|
|
|
// Part is one part of a transfer: its digest (sha256:<hex>), its size and how to read its bytes.
|
|
type Part struct {
|
|
Digest string
|
|
Size int64
|
|
Open func() (io.ReadCloser, error)
|
|
}
|
|
|
|
// Opened is the receiver's answer to opening a transfer: whether it holds the whole already, and the digests of the
|
|
// parts it lacks.
|
|
type Opened struct {
|
|
Held bool
|
|
Lacks []string
|
|
}
|
|
|
|
// Open opens (or opens again) the transfer on the receiver, naming every part: the caller's own call.
|
|
type Open func(ctx context.Context, parts []Part) (Opened, error)
|
|
|
|
// Chunk sends one chunk of a part: the part's digest, its sequence from 0, whether it is the part's last, its bytes.
|
|
type Chunk func(ctx context.Context, part string, sequence int, last bool, data []byte) error
|
|
|
|
// Sent is what a Send did: whether the receiver held the whole, the parts it lacked and were sent, the chunks they
|
|
// took.
|
|
type Sent struct {
|
|
Held bool
|
|
Parts []string
|
|
Chunks int
|
|
}
|
|
|
|
// Send opens the transfer and sends the parts the receiver answers it lacks, each in chunks of at most MaxChunk, in
|
|
// sequence, the last marked. A part whose bytes do not hash to its digest is refused before anything of it is sent; a
|
|
// part the receiver does not name is never read; a part the receiver names that the transfer does not is refused.
|
|
func Send(ctx context.Context, open Open, chunk Chunk, parts []Part) (Sent, error) {
|
|
byDigest := map[string]Part{}
|
|
for _, p := range parts {
|
|
byDigest[p.Digest] = p
|
|
}
|
|
opened, err := open(ctx, parts)
|
|
if err != nil {
|
|
return Sent{}, err
|
|
}
|
|
sent := Sent{Held: opened.Held}
|
|
for _, d := range opened.Lacks {
|
|
p, ok := byDigest[d]
|
|
if !ok {
|
|
return sent, fmt.Errorf("the receiver lacks the part %s, which the transfer does not name", d)
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return sent, err
|
|
}
|
|
if err := check(p); err != nil {
|
|
return sent, err
|
|
}
|
|
n, err := sendPart(ctx, chunk, p)
|
|
sent.Chunks += n
|
|
if err != nil {
|
|
return sent, err
|
|
}
|
|
sent.Parts = append(sent.Parts, d)
|
|
}
|
|
return sent, nil
|
|
}
|
|
|
|
// check reads the part once and refuses it when its bytes are not its digest or its size.
|
|
func check(p Part) error {
|
|
r, err := p.Open()
|
|
if err != nil {
|
|
return fmt.Errorf("the part %s cannot be read: %w", p.Digest, err)
|
|
}
|
|
defer r.Close()
|
|
h := sha256.New()
|
|
n, err := io.Copy(h, r)
|
|
if err != nil {
|
|
return fmt.Errorf("the part %s cannot be read: %w", p.Digest, err)
|
|
}
|
|
if got := "sha256:" + hex.EncodeToString(h.Sum(nil)); got != p.Digest || n != p.Size {
|
|
return fmt.Errorf("the part %s holds %d bytes of digest %s, not its own %d: nothing of it is sent", p.Digest, n, got, p.Size)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func sendPart(ctx context.Context, chunk Chunk, p Part) (int, error) {
|
|
r, err := p.Open()
|
|
if err != nil {
|
|
return 0, fmt.Errorf("the part %s cannot be read: %w", p.Digest, err)
|
|
}
|
|
defer r.Close()
|
|
buf := make([]byte, MaxChunk)
|
|
// A chunk is sent once the next read says whether it is the last, so the last is always marked: an empty part is
|
|
// one empty chunk, marked last.
|
|
var prev []byte
|
|
sent := 0
|
|
send := func(data []byte, last bool) error {
|
|
if err := chunk(ctx, p.Digest, sent, last, data); err != nil {
|
|
return fmt.Errorf("chunk %d of the part %s: %w", sent, p.Digest, err)
|
|
}
|
|
sent++
|
|
return nil
|
|
}
|
|
for {
|
|
if err := ctx.Err(); err != nil {
|
|
return sent, err
|
|
}
|
|
n, err := io.ReadFull(r, buf)
|
|
end := errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF)
|
|
if err != nil && !end {
|
|
return sent, fmt.Errorf("the part %s cannot be read: %w", p.Digest, err)
|
|
}
|
|
if end && n == 0 {
|
|
if prev == nil {
|
|
prev = []byte{}
|
|
}
|
|
return sent, send(prev, true)
|
|
}
|
|
if prev != nil {
|
|
if err := send(prev, false); err != nil {
|
|
return sent, err
|
|
}
|
|
}
|
|
prev = append([]byte(nil), buf[:n]...)
|
|
if end {
|
|
return sent, send(prev, true)
|
|
}
|
|
}
|
|
}
|