// Package objectlink signs and streams a link to one object in the mesh's object store (novox/hq ADR 0468 §3). // // Bytes move over HTTP between a party and the store, on a signed link of at most ten minutes for one object and one // method, signed for the name the `s3-bucket` binding gives; a document over 100 MiB is refused. Presign makes the // link (Signature Version 4, query form, path style, payload unsigned, `host` the only signed header); Put and Get // move the bytes without holding them in memory and count and hash them as they go. // // Standard library only: the SDK requires nothing of the mesh's forge. package objectlink import ( "context" "crypto/hmac" "crypto/sha256" "encoding/hex" "encoding/xml" "errors" "fmt" "io" "net/http" "net/url" "sort" "strings" "time" ) const ( // MaxDocument is the largest document a link moves (ADR 0468 §3): the default limit of Put and Get. MaxDocument int64 = 100 << 20 // MaxExpires is the longest a link lives (ADR 0468 §3). MaxExpires = 10 * time.Minute defaultRegion = "us-east-1" algorithm = "AWS4-HMAC-SHA256" ) // Request is one link to sign. type Request struct { // Method is GET or PUT. Method string // Endpoint is the store as the party reaches it, "http://host:port". Its host is signed exactly as written. Endpoint string Bucket string Key string // AccessKey and SecretKey are the credential the s3-bucket binding gives. AccessKey string SecretKey string // Region defaults to us-east-1. Region string // Expires is how long the link lives: at least one second, at most ten minutes. Expires time.Duration // Now is the signing time; the clock's when zero. Now time.Time } // Presign returns the signed link for one object and one method. func Presign(r Request) (string, error) { if r.Method != http.MethodGet && r.Method != http.MethodPut { return "", fmt.Errorf("objectlink: method %q: a link is for GET or PUT (ADR 0468 §3)", r.Method) } if r.Expires < time.Second || r.Expires > MaxExpires { return "", fmt.Errorf("objectlink: a link lives at least 1s and at most %s, not %s (ADR 0468 §3)", MaxExpires, r.Expires) } if r.Bucket == "" || r.Key == "" || r.AccessKey == "" || r.SecretKey == "" { return "", errors.New("objectlink: a bucket, a key and a credential are needed to sign a link") } u, err := url.Parse(r.Endpoint) if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") { return "", fmt.Errorf("objectlink: endpoint %q is not http(s)://host[:port]", r.Endpoint) } region := r.Region if region == "" { region = defaultRegion } now := r.Now if now.IsZero() { now = time.Now() } path := "/" + encode(r.Bucket, false) + "/" + encode(r.Key, false) query := sign(r.Method, u.Host, path, r.AccessKey, r.SecretKey, region, now.UTC(), int64(r.Expires/time.Second)) return u.Scheme + "://" + u.Host + path + "?" + query, nil } // sign returns the query string of a presigned link, signature last. The path is already encoded. func sign(method, host, path, accessKey, secretKey, region string, now time.Time, expires int64) string { date := now.Format("20060102") stamp := now.Format("20060102T150405Z") scope := date + "/" + region + "/s3/aws4_request" params := map[string]string{ "X-Amz-Algorithm": algorithm, "X-Amz-Credential": accessKey + "/" + scope, "X-Amz-Date": stamp, "X-Amz-Expires": fmt.Sprint(expires), "X-Amz-SignedHeaders": "host", } names := make([]string, 0, len(params)) for k := range params { names = append(names, k) } sort.Strings(names) var query []string for _, k := range names { query = append(query, encode(k, true)+"="+encode(params[k], true)) } canonicalQuery := strings.Join(query, "&") canonical := strings.Join([]string{method, path, canonicalQuery, "host:" + host + "\n", "host", "UNSIGNED-PAYLOAD"}, "\n") digest := sha256.Sum256([]byte(canonical)) toSign := strings.Join([]string{algorithm, stamp, scope, hex.EncodeToString(digest[:])}, "\n") key := mac([]byte("AWS4"+secretKey), date) key = mac(key, region) key = mac(key, "s3") key = mac(key, "aws4_request") return canonicalQuery + "&X-Amz-Signature=" + hex.EncodeToString(mac(key, toSign)) } func mac(key []byte, data string) []byte { h := hmac.New(sha256.New, key) h.Write([]byte(data)) return h.Sum(nil) } // encode is AWS's URI encoding: everything but A-Z a-z 0-9 - _ . ~ as %XX, and the slash kept in a path. func encode(s string, slash bool) string { var b strings.Builder for i := 0; i < len(s); i++ { c := s[i] switch { case c >= 'A' && c <= 'Z', c >= 'a' && c <= 'z', c >= '0' && c <= '9', c == '-', c == '_', c == '.', c == '~': b.WriteByte(c) case c == '/' && !slash: b.WriteByte(c) default: fmt.Fprintf(&b, "%%%02X", c) } } return b.String() } // Result is what moved: its size and the SHA-256 of its bytes, in hex. type Result struct { Size int64 SHA256 string } // StoreError is a refusal by the store: its status and the code and message of its XML error body. type StoreError struct { Status int Code string Message string } func (e *StoreError) Error() string { if e.Code == "" { return fmt.Sprintf("objectlink: the store answered %d", e.Status) } return fmt.Sprintf("objectlink: the store answered %d %s: %s", e.Status, e.Code, e.Message) } func storeError(resp *http.Response) error { body, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<10)) var parsed struct { Code string `xml:"Code"` Message string `xml:"Message"` } _ = xml.Unmarshal(body, &parsed) return &StoreError{Status: resp.StatusCode, Code: parsed.Code, Message: parsed.Message} } // tooBig is the error of a document past its limit. func tooBig(limit int64) error { return fmt.Errorf("objectlink: the document is over %d bytes (ADR 0468 §3)", limit) } // exact counts and hashes what passes, and fails a stream that is longer than declared, or ends before it. type exact struct { r io.Reader size int64 n int64 hash interface{ Write([]byte) (int, error) } } func (l *exact) Read(p []byte) (int, error) { n, err := l.r.Read(p) l.n += int64(n) if l.n > l.size { return 0, fmt.Errorf("objectlink: the body is longer than the declared %d bytes", l.size) } l.hash.Write(p[:n]) if err == io.EOF && l.n < l.size { return n, fmt.Errorf("objectlink: the body ended at %d bytes, short of the declared %d", l.n, l.size) } return n, err } // Put streams body to a link made for PUT, never holding it in memory. The caller gives the body's exact size, which // is always sent as the Content-Length (a store answers 411 to a PUT without one). A negative size is refused, and so // is one over limit (MaxDocument when limit is not above zero, ADR 0468 §3), both before any request. A body that // yields more or fewer bytes than size fails the request, so a PUT whose body does not end as declared stores nothing. func Put(ctx context.Context, client *http.Client, link string, body io.Reader, size, limit int64) (Result, error) { if limit <= 0 { limit = MaxDocument } if client == nil { client = http.DefaultClient } if size < 0 { return Result{}, errors.New("objectlink: put needs the size of the body; a negative or unknown size is refused") } if size > limit { return Result{}, tooBig(limit) } hash := sha256.New() ctx, cancel := context.WithCancel(ctx) defer cancel() counted := &exact{r: body, size: size, hash: hash} req, err := http.NewRequestWithContext(ctx, http.MethodPut, link, io.NopCloser(counted)) if err != nil { return Result{}, err } req.ContentLength = size if size == 0 { req.Body = http.NoBody } resp, err := client.Do(req) if err != nil { cancel() return Result{}, fmt.Errorf("objectlink: put: %w", err) } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode > 299 { return Result{}, storeError(resp) } if counted.n != size { return Result{}, fmt.Errorf("objectlink: put: sent %d bytes of the declared %d", counted.n, size) } return Result{Size: counted.n, SHA256: hex.EncodeToString(hash.Sum(nil))}, nil } // Get streams the object of a link made for GET into w, refusing more than limit bytes (MaxDocument when limit is // not above zero). When wantSHA256 is given and the bytes hash to another value the call fails: the caller removes // what it wrote. func Get(ctx context.Context, client *http.Client, link string, w io.Writer, limit int64, wantSHA256 string) (Result, error) { if limit <= 0 { limit = MaxDocument } if client == nil { client = http.DefaultClient } req, err := http.NewRequestWithContext(ctx, http.MethodGet, link, nil) if err != nil { return Result{}, err } resp, err := client.Do(req) if err != nil { return Result{}, fmt.Errorf("objectlink: get: %w", err) } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode > 299 { return Result{}, storeError(resp) } if resp.ContentLength > limit { return Result{}, tooBig(limit) } hash := sha256.New() n, err := io.Copy(io.MultiWriter(w, hash), io.LimitReader(resp.Body, limit+1)) if err != nil { return Result{}, fmt.Errorf("objectlink: get: %w", err) } if n > limit { return Result{}, tooBig(limit) } got := hex.EncodeToString(hash.Sum(nil)) if wantSHA256 != "" && !strings.EqualFold(wantSHA256, got) { return Result{}, fmt.Errorf("objectlink: the document hashes to %s, not the %s expected", got, strings.ToLower(wantSHA256)) } return Result{Size: n, SHA256: got}, nil }