Merge pull request 'Name a bucket's other users in the manifest, and sign and stream store links in one helper (issues 1591, 1592)' (#54) from feat/files-through-the-store into main

go/catalogue/bucket_shares.go

@@ -0,0 +1,124 @@
+package catalogue
+
+import (
+	"encoding/json"
+	"fmt"
+	"regexp"
+	"sort"
+)
+
+// BucketShare is one entry of `shares` on a module's requirement of `s3-bucket` (novox/hq ADR 0468 §8): who else may
+// put or get in the consumer's bucket, and with which methods. Not the NFS shares of shares.go (ADR 0263).
+//
+//	"requires": [{"name": "s3-bucket", "shares": [{"to": "seat:node-files", "methods": ["put"]},
+//	                                             {"to": "print-and-scan", "methods": ["get"]}]}]
+//
+// `to` is a module name, or `seat:<seat name>`. `methods` is a non-empty subset of get and put, without a repeat.
+// Shares are allowed on `s3-bucket` only, and no two entries name the same `to`.
+type BucketShare struct {
+	To      string   `json:"to"`
+	Methods []string `json:"methods"`
+}
+
+// BucketRequirement is the requirement `shares` may be written on.
+const BucketRequirement = "s3-bucket"
+
+// RequireSharesKey is where a stored manifest keeps the shares of its requirements: beside `requires`, which stays a
+// plain list of names, for the reason OptionalRequiresKey is beside it. An older controller sets the unknown key
+// aside instead of failing the catalogue.
+const RequireSharesKey = "requires-shares"
+
+// SharesValue is the key of the value a consumer gives the provider of s3-bucket in its grant: the list of
+// {to, methods} sorted by `to`. A consumer's own `contributes` may not name it.
+const SharesValue = "shares"
+
+var shareTarget = regexp.MustCompile(`^(seat:)?[a-z0-9][a-z0-9-]*$`)
+
+// checkBucketShares refuses a malformed `shares` of one requirement, naming the entry and ADR 0468 §8.
+func checkBucketShares(requirement string, shares []BucketShare) error {
+	if len(shares) == 0 {
+		return nil
+	}
+	if requirement != BucketRequirement {
+		return fmt.Errorf("requires %s: `shares` is allowed on %s only (novox/hq ADR 0468 §8)", requirement, BucketRequirement)
+	}
+	seen := map[string]bool{}
+	for i, s := range shares {
+		at := fmt.Sprintf("requires %s: shares[%d] (to %q)", requirement, i, s.To)
+		if !shareTarget.MatchString(s.To) {
+			return fmt.Errorf("%s: `to` is a module name or seat:<seat name> (novox/hq ADR 0468 §8)", at)
+		}
+		if seen[s.To] {
+			return fmt.Errorf("%s: a second entry for the same `to` (novox/hq ADR 0468 §8)", at)
+		}
+		seen[s.To] = true
+		if len(s.Methods) == 0 {
+			return fmt.Errorf("%s: `methods` is empty; name get, put or both (novox/hq ADR 0468 §8)", at)
+		}
+		used := map[string]bool{}
+		for _, method := range s.Methods {
+			if method != "get" && method != "put" {
+				return fmt.Errorf("%s: method %q is not get or put (novox/hq ADR 0468 §8)", at, method)
+			}
+			if used[method] {
+				return fmt.Errorf("%s: method %q twice (novox/hq ADR 0468 §8)", at, method)
+			}
+			used[method] = true
+		}
+	}
+	return nil
+}
+
+// readRequireShares reads the stored form's side key.
+func readRequireShares(raw json.RawMessage) (map[string][]BucketShare, error) {
+	var out map[string][]BucketShare
+	if err := json.Unmarshal(raw, &out); err != nil {
+		return nil, fmt.Errorf("%s: an object of requirement to a list of {to, methods}: %w", RequireSharesKey, err)
+	}
+	for requirement, shares := range out {
+		if err := checkBucketShares(requirement, shares); err != nil {
+			return nil, err
+		}
+	}
+	return out, nil
+}
+
+// sortedShares is a requirement's shares as they are handed to the provider: sorted by `to` (a file whose lines move
+// looks changed), each entry's methods sorted, and a copy so the manifest is not reordered.
+func sortedShares(shares []BucketShare) []BucketShare {
+	out := make([]BucketShare, 0, len(shares))
+	for _, s := range shares {
+		methods := append([]string(nil), s.Methods...)
+		sort.Strings(methods)
+		out = append(out, BucketShare{To: s.To, Methods: methods})
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].To < out[j].To })
+	return out
+}
+
+// withShares adds this module's shares of the requirement to the values it gives the provider. A module whose own
+// `contributes` already names `shares` is refused: the field is the only way to say it (ADR 0468 §8).
+func withShares(m Manifest, requirement string, values map[string]any) (map[string]any, error) {
+	shares := m.RequireShares[requirement]
+	if _, named := values[SharesValue]; named && requirement == BucketRequirement {
+		return nil, fmt.Errorf("%s contributing to %s names `shares`; write it on its requirement instead (novox/hq ADR 0468 §8)",
+			m.Module, requirement)
+	}
+	if len(shares) == 0 {
+		return values, nil
+	}
+	out := make(map[string]any, len(values)+1)
+	for k, v := range values {
+		out[k] = v
+	}
+	listed := []any{}
+	for _, s := range sortedShares(shares) {
+		methods := []any{}
+		for _, method := range s.Methods {
+			methods = append(methods, method)
+		}
+		listed = append(listed, map[string]any{"to": s.To, "methods": methods})
+	}
+	out[SharesValue] = listed
+	return out, nil
+}

go/catalogue/bucket_shares_test.go

@@ -0,0 +1,185 @@
+package catalogue
+
+import (
+	"encoding/json"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+// Issue 1591: `shares` on a requirement of s3-bucket (novox/hq ADR 0468 §8).
+
+const filesWithShares = `{"module":"files","version":"1","requires":["shell",
+	{"name":"s3-bucket","shares":[{"to":"seat:node-files","methods":["put"]},{"to":"print-and-scan","methods":["get","put"]}]}]}`
+
+func minioLike() Manifest {
+	return Manifest{Module: "minio", Version: "1", Provides: Offers("s3-bucket"),
+		Receives: map[string]string{"s3-bucket": "/grants/mesh.json"}}
+}
+
+func givenToMinio(t *testing.T, out []map[string]any) []Contribution {
+	t.Helper()
+	for _, r := range out {
+		if r["path"] != "/grants/mesh.json" {
+			continue
+		}
+		var parsed struct {
+			Given []Contribution `json:"given"`
+		}
+		if err := json.Unmarshal([]byte(r["content"].(string)), &parsed); err != nil {
+			t.Fatal(err)
+		}
+		return parsed.Given
+	}
+	t.Fatalf("minio was given no file: %v", out)
+	return nil
+}
+
+func sharesOf(t *testing.T, values map[string]any) string {
+	t.Helper()
+	raw, err := json.Marshal(values["shares"])
+	if err != nil {
+		t.Fatal(err)
+	}
+	return string(raw)
+}
+
+const sortedSharesJSON = `[{"methods":["get","put"],"to":"print-and-scan"},{"methods":["put"],"to":"seat:node-files"}]`
+
+// 1591/AC1: parsed, the shares are on the manifest; stored, `requires` is plain names and the side key holds them.
+func TestSharesParseAndAreStoredBesideRequires(t *testing.T) {
+	m := mustParse(t, filesWithShares)
+	want := []BucketShare{{To: "seat:node-files", Methods: []string{"put"}}, {To: "print-and-scan", Methods: []string{"get", "put"}}}
+	if !reflect.DeepEqual(m.RequireShares["s3-bucket"], want) {
+		t.Fatalf("the shares read as %+v", m.RequireShares)
+	}
+	raw, err := json.Marshal(m)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var keys map[string]json.RawMessage
+	if err := json.Unmarshal(raw, &keys); err != nil {
+		t.Fatal(err)
+	}
+	if string(keys["requires"]) != `["shell","s3-bucket"]` {
+		t.Errorf("the stored requires is not plain names: %s", keys["requires"])
+	}
+	if !strings.Contains(string(keys[RequireSharesKey]), `"seat:node-files"`) {
+		t.Errorf("the side key does not hold the shares: %s", raw)
+	}
+	var again Manifest
+	if err := json.Unmarshal(raw, &again); err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(again, m) {
+		t.Fatalf("not lossless:\n%+v\n%+v", again, m)
+	}
+}
+
+// 1591/AC2: a malformed share is refused, naming the entry and ADR 0468 §8.
+func TestAMalformedShareIsRefused(t *testing.T) {
+	cases := map[string]string{
+		"unknown method": `{"name":"s3-bucket","shares":[{"to":"a","methods":["delete"]}]}`,
+		"empty to":       `{"name":"s3-bucket","shares":[{"to":"","methods":["get"]}]}`,
+		"malformed to":   `{"name":"s3-bucket","shares":[{"to":"Seat:X y","methods":["get"]}]}`,
+		"empty methods":  `{"name":"s3-bucket","shares":[{"to":"a","methods":[]}]}`,
+		"duplicate to":   `{"name":"s3-bucket","shares":[{"to":"a","methods":["get"]},{"to":"a","methods":["put"]}]}`,
+		"duplicate verb": `{"name":"s3-bucket","shares":[{"to":"a","methods":["get","get"]}]}`,
+		"other require":  `{"name":"route","shares":[{"to":"a","methods":["get"]}]}`,
+	}
+	for name, entry := range cases {
+		_, err := ParseManifest([]byte(`{"module":"files","version":"1","requires":[` + entry + `]}`))
+		if err == nil {
+			t.Errorf("%s: accepted", name)
+			continue
+		}
+		if !strings.Contains(err.Error(), "ADR 0468 §8") || !strings.Contains(err.Error(), "requires") {
+			t.Errorf("%s: the error does not name the entry and the ADR: %v", name, err)
+		}
+	}
+	// The stored form is held to the same rules.
+	_, err := ParseManifest([]byte(`{"module":"files","version":"1","requires":["s3-bucket"],
+		"requires-shares":{"s3-bucket":[{"to":"a","methods":[]}]}}`))
+	if err == nil || !strings.Contains(err.Error(), "ADR 0468 §8") {
+		t.Errorf("the stored form was accepted: %v", err)
+	}
+}
+
+// 1591/AC3: the values a consumer gives carry the shares sorted by `to`, remote (grant) and co-located alike.
+func TestTheGrantCarriesTheSharesSorted(t *testing.T) {
+	m := mustParse(t, filesWithShares)
+	remote := Resolution{Node: "laptop", Modules: []Manifest{m}}
+	values, asks, err := remote.ContributionsFrom("s3-bucket", "files", SettingsBy{})
+	if err != nil || !asks {
+		t.Fatalf("asks %v, err %v", asks, err)
+	}
+	if got := sharesOf(t, values); got != sortedSharesJSON {
+		t.Fatalf("the grant values carry %s", got)
+	}
+	// As the provider's file shows it, through a grant.
+	provider, err := Resolve(shelf(minioLike()), []string{"minio"}, workstation(), World{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	out, err := provider.Declaration(Rendering{Grants: []Grant{{Provision: "s3-bucket", Consumer: "laptop", From: "files", Values: values}}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	given := givenToMinio(t, out)
+	if len(given) != 1 || sharesOf(t, given[0].Values) != sortedSharesJSON {
+		t.Fatalf("the remote consumer's shares did not reach the file: %+v", given)
+	}
+
+	// Co-located: the consumer contributes to s3-bucket too, and no grant is passed.
+	co := mustParse(t, `{"module":"files","version":"1","contributes":{"s3-bucket":{"name":"files"}},
+		"requires":[{"name":"s3-bucket","shares":[{"to":"seat:node-files","methods":["put"]},{"to":"print-and-scan","methods":["put","get"]}]}]}`)
+	both, err := Resolve(shelf(minioLike(), co), []string{"files"}, workstation(), World{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	out, err = both.Declaration(Rendering{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	given = givenToMinio(t, out)
+	if len(given) != 1 || sharesOf(t, given[0].Values) != sortedSharesJSON || given[0].Values["name"] != "files" {
+		t.Fatalf("the co-located consumer's shares did not reach the file: %+v", given)
+	}
+}
+
+// 1591/AC4: only the shares change, so the module is not recreated and held for no one; the provider's file changes.
+func TestChangingOnlyTheSharesDoesNotRecreateTheConsumer(t *testing.T) {
+	was := mustParse(t, filesWithShares)
+	now := mustParse(t, `{"module":"files","version":"1","requires":["shell",
+		{"name":"s3-bucket","shares":[{"to":"print-and-scan","methods":["get"]}]}]}`)
+	if said := Recreates(was, now).Say("files"); said != "" {
+		t.Errorf("a change of shares recreates the module: %s", said)
+	}
+	if said := Holds(was, now); said != "" {
+		t.Errorf("a change of shares holds the module: %s", said)
+	}
+	before := Resolution{Node: "laptop", Modules: []Manifest{was}}
+	after := Resolution{Node: "laptop", Modules: []Manifest{now}}
+	a, _, _ := before.ContributionsFrom("s3-bucket", "files", SettingsBy{})
+	b, _, _ := after.ContributionsFrom("s3-bucket", "files", SettingsBy{})
+	if sharesOf(t, a) == sharesOf(t, b) {
+		t.Fatalf("the provider's values did not change: %s", sharesOf(t, a))
+	}
+}
+
+// 1591/AC5: a consumer whose contributes to s3-bucket names `shares` itself is refused.
+func TestAConsumerCannotContributeSharesItself(t *testing.T) {
+	_, err := ParseManifest([]byte(`{"module":"files","version":"1",
+		"contributes":{"s3-bucket":{"shares":"x"}},
+		"requires":[{"name":"s3-bucket","shares":[{"to":"a","methods":["get"]}]}]}`))
+	if err == nil || !strings.Contains(err.Error(), "ADR 0468 §8") {
+		t.Errorf("both written, accepted: %v", err)
+	}
+	// Without the field the grant is still refused, so the provider is never told something the manifest did not say.
+	m := Manifest{Module: "files", Requires: []string{"s3-bucket"},
+		Contributes: map[string]map[string]any{"s3-bucket": {"shares": "x"}}}
+	r := Resolution{Node: "laptop", Modules: []Manifest{m}}
+	if _, _, err := r.ContributionsFrom("s3-bucket", "files", SettingsBy{}); err == nil {
+		t.Error("a contributed `shares` reached the grant")
+	}
+}

go/catalogue/declaration.go

@@ -1783,6 +1783,10 @@ func (r Resolution) contributions(settings SettingsBy, grants []Grant,
 			if err != nil {
 				return nil, err
 			}
+			// What the module shares of its bucket reaches the provider with the rest (novox/hq ADR 0468 §8).
+			if values, err = withShares(m, to, values); err != nil {
+				return nil, err
+			}
 			out[to] = append(out[to], Contribution{From: m.Module, Values: values})
 		}
 		// Several contributions to one requirement (ADR 0094's sibling for `contributes`): an
@@ -2169,11 +2173,19 @@ func (r Resolution) ContributionsFrom(requirement, module string, settings Setti
 			mine = append(mine, g.Values)
 		}
 	}
+	var asker Manifest
+	for _, m := range r.Modules {
+		if m.Module == module {
+			asker = m
+		}
+	}
 	if len(mine) == 1 {
+		// Its shares are already in, put there where its contribution was composed (contributions).
 		return mine[0], true, nil
 	}
 	if len(mine) > 1 {
-		return map[string]any{}, true, nil
+		values, err := withShares(asker, requirement, map[string]any{})
+		return values, err == nil, err
 	}
 	// It contributes no payload — but a require-only consumer of a parameterless provision (one whose
 	// `serves` names no consumer-supplied key: `redis-cache`, `amqp`) still ASKS for it and must be
@@ -2186,7 +2198,8 @@ func (r Resolution) ContributionsFrom(requirement, module string, settings Setti
 		}
 		for _, req := range m.Requires {
 			if req == requirement {
-				return map[string]any{}, true, nil
+				values, err := withShares(m, requirement, map[string]any{})
+				return values, err == nil, err
 			}
 		}
 	}

go/catalogue/manifest.go

@@ -494,6 +494,9 @@ type Manifest struct {
 	// by UnmarshalJSON from the long form of a `requires` entry, {name, optional}, or from the stored
 	// form's `optional-requires` list; never written by hand.
 	OptionalRequires map[string]bool `json:"-"`
+	// RequireShares is who else may use the bucket a module requires, by requirement (BucketShare, novox/hq ADR 0468
+	// §8): filled from the long form's `shares`, or from the stored form's `requires-shares`; never written by hand.
+	RequireShares map[string][]BucketShare `json:"-"`
 
 	// Emits are the events this module publishes, named **locally**: `order.placed`, not a subject
 	// and not a routing key. The mesh derives where it lands (design 29 §1), so reorganising the
@@ -1538,8 +1541,9 @@ func (m *Manifest) UnmarshalJSON(raw []byte) error {
 		delete(keys, "contributes")
 	}
 	var optionalRequires map[string]bool
+	var requireShares map[string][]BucketShare
 	if requires, ok := keys["requires"]; ok && string(requires) != "null" {
-		names, optional, err := readRequires(requires)
+		names, optional, shares, err := readRequires(requires)
 		if err != nil {
 			return err
 		}
@@ -1549,6 +1553,23 @@ func (m *Manifest) UnmarshalJSON(raw []byte) error {
 		}
 		keys["requires"] = plainNames
 		optionalRequires = optional
+		requireShares = shares
+	}
+	// The stored form's shares (RequireSharesKey).
+	if stored, ok := keys[RequireSharesKey]; ok {
+		if string(stored) != "null" {
+			stood, err := readRequireShares(stored)
+			if err != nil {
+				return err
+			}
+			for requirement, shares := range stood {
+				if requireShares == nil {
+					requireShares = map[string][]BucketShare{}
+				}
+				requireShares[requirement] = shares
+			}
+		}
+		delete(keys, RequireSharesKey)
 	}
 	// The stored form: `requires` plain, and which are optional beside it (OptionalRequiresKey).
 	if stored, ok := keys[OptionalRequiresKey]; ok {
@@ -1599,6 +1620,7 @@ func (m *Manifest) UnmarshalJSON(raw []byte) error {
 	m.unknown = unknown
 	m.forms = forms
 	m.OptionalRequires = optionalRequires
+	m.RequireShares = requireShares
 	if len(plain) > 0 {
 		m.Secrets = plain
 	}
@@ -1611,6 +1633,21 @@ func (m *Manifest) UnmarshalJSON(raw []byte) error {
 	if len(contributesMany) > 0 {
 		m.ContributesMany = contributesMany
 	}
+	if err := m.checkSharesNotContributed(); err != nil {
+		return err
+	}
+	return nil
+}
+
+// checkSharesNotContributed refuses a module whose own `contributes` to s3-bucket names `shares` (ADR 0468 §8).
+func (m Manifest) checkSharesNotContributed() error {
+	if len(m.RequireShares[BucketRequirement]) == 0 {
+		return nil
+	}
+	if _, named := m.Contributes[BucketRequirement][SharesValue]; named {
+		return fmt.Errorf("contributes.%s names `shares` and so does its requirement; keep one, the requirement's (novox/hq ADR 0468 §8)",
+			BucketRequirement)
+	}
 	return nil
 }
 
@@ -1655,7 +1692,7 @@ func (m Manifest) marshalFields() ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
-	if len(m.SecretsMany) == 0 && len(m.ContributesMany) == 0 && len(m.OptionalRequires) == 0 {
+	if len(m.SecretsMany) == 0 && len(m.ContributesMany) == 0 && len(m.OptionalRequires) == 0 && len(m.RequireShares) == 0 {
 		return raw, nil
 	}
 	var keys map[string]json.RawMessage
@@ -1671,6 +1708,13 @@ func (m Manifest) marshalFields() ([]byte, error) {
 		}
 		keys[OptionalRequiresKey] = listed
 	}
+	if len(m.RequireShares) > 0 {
+		listed, err := json.Marshal(m.RequireShares)
+		if err != nil {
+			return nil, err
+		}
+		keys[RequireSharesKey] = listed
+	}
 	if len(m.ContributesMany) > 0 {
 		mergedContributes := map[string]any{}
 		for to, values := range m.Contributes {

go/catalogue/optional_requirements.go

@@ -32,17 +32,20 @@ const OnlyWhenBound = "only-when-bound"
 type requirementForm struct {
 	Name     string `json:"name"`
 	Optional bool   `json:"optional,omitempty"`
+	// Shares is who else may use the consumer's bucket (BucketShare, novox/hq ADR 0468 §8), on s3-bucket only.
+	Shares []BucketShare `json:"shares,omitempty"`
 }
 
 // readRequires reads `requires` in both of its forms — a bare name, or {name, optional} — into the
 // names and which of them are optional.
-func readRequires(raw json.RawMessage) ([]string, map[string]bool, error) {
+func readRequires(raw json.RawMessage) ([]string, map[string]bool, map[string][]BucketShare, error) {
 	var entries []json.RawMessage
 	if err := json.Unmarshal(raw, &entries); err != nil {
-		return nil, nil, fmt.Errorf("requires: a list of names, or of {name, optional}: %w", err)
+		return nil, nil, nil, fmt.Errorf("requires: a list of names, or of {name, optional}: %w", err)
 	}
 	var names []string
 	optional := map[string]bool{}
+	var shares map[string][]BucketShare
 	for i, e := range entries {
 		var plain string
 		if err := json.Unmarshal(e, &plain); err == nil {
@@ -53,12 +56,21 @@ func readRequires(raw json.RawMessage) ([]string, map[string]bool, error) {
 		dec := json.NewDecoder(bytes.NewReader(e))
 		dec.DisallowUnknownFields()
 		if err := dec.Decode(&full); err != nil {
-			return nil, nil, fmt.Errorf("requires[%d]: a name, or {name, optional} (novox/hq ADR 0330): %w", i, err)
+			return nil, nil, nil, fmt.Errorf("requires[%d]: a name, or {name, optional} (novox/hq ADR 0330): %w", i, err)
 		}
 		if strings.TrimSpace(full.Name) == "" {
-			return nil, nil, fmt.Errorf("requires[%d] names no requirement: {name, optional}", i)
+			return nil, nil, nil, fmt.Errorf("requires[%d] names no requirement: {name, optional}", i)
 		}
 		names = append(names, full.Name)
+		if err := checkBucketShares(full.Name, full.Shares); err != nil {
+			return nil, nil, nil, fmt.Errorf("requires[%d]: %w", i, err)
+		}
+		if len(full.Shares) > 0 {
+			if shares == nil {
+				shares = map[string][]BucketShare{}
+			}
+			shares[full.Name] = full.Shares
+		}
 		if full.Optional {
 			optional[full.Name] = true
 		}
@@ -66,7 +78,7 @@ func readRequires(raw json.RawMessage) ([]string, map[string]bool, error) {
 	if len(optional) == 0 {
 		optional = nil
 	}
-	return names, optional, nil
+	return names, optional, shares, nil
 }
 
 // OptionalRequiresKey is where a stored manifest keeps which of its requirements are optional: beside

go/objectlink/objectlink.go

@@ -0,0 +1,277 @@
+// Package objectlink signs and streams a link to one object in the mesh's object store (novox/hq ADR 0468 §3).
+//
+// Bytes move over HTTP between a party and the store, on a signed link of at most ten minutes for one object and one
+// method, signed for the name the `s3-bucket` binding gives; a document over 100 MiB is refused. Presign makes the
+// link (Signature Version 4, query form, path style, payload unsigned, `host` the only signed header); Put and Get
+// move the bytes without holding them in memory and count and hash them as they go.
+//
+// Standard library only: the SDK requires nothing of the mesh's forge.
+package objectlink
+
+import (
+	"context"
+	"crypto/hmac"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"sort"
+	"strings"
+	"time"
+)
+
+const (
+	// MaxDocument is the largest document a link moves (ADR 0468 §3): the default limit of Put and Get.
+	MaxDocument int64 = 100 << 20
+	// MaxExpires is the longest a link lives (ADR 0468 §3).
+	MaxExpires = 10 * time.Minute
+
+	defaultRegion = "us-east-1"
+	algorithm     = "AWS4-HMAC-SHA256"
+)
+
+// Request is one link to sign.
+type Request struct {
+	// Method is GET or PUT.
+	Method string
+	// Endpoint is the store as the party reaches it, "http://host:port". Its host is signed exactly as written.
+	Endpoint string
+	Bucket   string
+	Key      string
+	// AccessKey and SecretKey are the credential the s3-bucket binding gives.
+	AccessKey string
+	SecretKey string
+	// Region defaults to us-east-1.
+	Region string
+	// Expires is how long the link lives: at least one second, at most ten minutes.
+	Expires time.Duration
+	// Now is the signing time; the clock's when zero.
+	Now time.Time
+}
+
+// Presign returns the signed link for one object and one method.
+func Presign(r Request) (string, error) {
+	if r.Method != http.MethodGet && r.Method != http.MethodPut {
+		return "", fmt.Errorf("objectlink: method %q: a link is for GET or PUT (ADR 0468 §3)", r.Method)
+	}
+	if r.Expires < time.Second || r.Expires > MaxExpires {
+		return "", fmt.Errorf("objectlink: a link lives at least 1s and at most %s, not %s (ADR 0468 §3)", MaxExpires, r.Expires)
+	}
+	if r.Bucket == "" || r.Key == "" || r.AccessKey == "" || r.SecretKey == "" {
+		return "", errors.New("objectlink: a bucket, a key and a credential are needed to sign a link")
+	}
+	u, err := url.Parse(r.Endpoint)
+	if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") {
+		return "", fmt.Errorf("objectlink: endpoint %q is not http(s)://host[:port]", r.Endpoint)
+	}
+	region := r.Region
+	if region == "" {
+		region = defaultRegion
+	}
+	now := r.Now
+	if now.IsZero() {
+		now = time.Now()
+	}
+	path := "/" + encode(r.Bucket, false) + "/" + encode(r.Key, false)
+	query := sign(r.Method, u.Host, path, r.AccessKey, r.SecretKey, region, now.UTC(), int64(r.Expires/time.Second))
+	return u.Scheme + "://" + u.Host + path + "?" + query, nil
+}
+
+// sign returns the query string of a presigned link, signature last. The path is already encoded.
+func sign(method, host, path, accessKey, secretKey, region string, now time.Time, expires int64) string {
+	date := now.Format("20060102")
+	stamp := now.Format("20060102T150405Z")
+	scope := date + "/" + region + "/s3/aws4_request"
+	params := map[string]string{
+		"X-Amz-Algorithm":     algorithm,
+		"X-Amz-Credential":    accessKey + "/" + scope,
+		"X-Amz-Date":          stamp,
+		"X-Amz-Expires":       fmt.Sprint(expires),
+		"X-Amz-SignedHeaders": "host",
+	}
+	names := make([]string, 0, len(params))
+	for k := range params {
+		names = append(names, k)
+	}
+	sort.Strings(names)
+	var query []string
+	for _, k := range names {
+		query = append(query, encode(k, true)+"="+encode(params[k], true))
+	}
+	canonicalQuery := strings.Join(query, "&")
+	canonical := strings.Join([]string{method, path, canonicalQuery, "host:" + host + "\n", "host", "UNSIGNED-PAYLOAD"}, "\n")
+	digest := sha256.Sum256([]byte(canonical))
+	toSign := strings.Join([]string{algorithm, stamp, scope, hex.EncodeToString(digest[:])}, "\n")
+	key := mac([]byte("AWS4"+secretKey), date)
+	key = mac(key, region)
+	key = mac(key, "s3")
+	key = mac(key, "aws4_request")
+	return canonicalQuery + "&X-Amz-Signature=" + hex.EncodeToString(mac(key, toSign))
+}
+
+func mac(key []byte, data string) []byte {
+	h := hmac.New(sha256.New, key)
+	h.Write([]byte(data))
+	return h.Sum(nil)
+}
+
+// encode is AWS's URI encoding: everything but A-Z a-z 0-9 - _ . ~ as %XX, and the slash kept in a path.
+func encode(s string, slash bool) string {
+	var b strings.Builder
+	for i := 0; i < len(s); i++ {
+		c := s[i]
+		switch {
+		case c >= 'A' && c <= 'Z', c >= 'a' && c <= 'z', c >= '0' && c <= '9', c == '-', c == '_', c == '.', c == '~':
+			b.WriteByte(c)
+		case c == '/' && !slash:
+			b.WriteByte(c)
+		default:
+			fmt.Fprintf(&b, "%%%02X", c)
+		}
+	}
+	return b.String()
+}
+
+// Result is what moved: its size and the SHA-256 of its bytes, in hex.
+type Result struct {
+	Size   int64
+	SHA256 string
+}
+
+// StoreError is a refusal by the store: its status and the code and message of its XML error body.
+type StoreError struct {
+	Status  int
+	Code    string
+	Message string
+}
+
+func (e *StoreError) Error() string {
+	if e.Code == "" {
+		return fmt.Sprintf("objectlink: the store answered %d", e.Status)
+	}
+	return fmt.Sprintf("objectlink: the store answered %d %s: %s", e.Status, e.Code, e.Message)
+}
+
+func storeError(resp *http.Response) error {
+	body, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<10))
+	var parsed struct {
+		Code    string `xml:"Code"`
+		Message string `xml:"Message"`
+	}
+	_ = xml.Unmarshal(body, &parsed)
+	return &StoreError{Status: resp.StatusCode, Code: parsed.Code, Message: parsed.Message}
+}
+
+// tooBig is the error of a document past its limit.
+func tooBig(limit int64) error {
+	return fmt.Errorf("objectlink: the document is over %d bytes (ADR 0468 §3)", limit)
+}
+
+// exact counts and hashes what passes, and fails a stream that is longer than declared, or ends before it.
+type exact struct {
+	r    io.Reader
+	size int64
+	n    int64
+	hash interface{ Write([]byte) (int, error) }
+}
+
+func (l *exact) Read(p []byte) (int, error) {
+	n, err := l.r.Read(p)
+	l.n += int64(n)
+	if l.n > l.size {
+		return 0, fmt.Errorf("objectlink: the body is longer than the declared %d bytes", l.size)
+	}
+	l.hash.Write(p[:n])
+	if err == io.EOF && l.n < l.size {
+		return n, fmt.Errorf("objectlink: the body ended at %d bytes, short of the declared %d", l.n, l.size)
+	}
+	return n, err
+}
+
+// Put streams body to a link made for PUT, never holding it in memory. The caller gives the body's exact size, which
+// is always sent as the Content-Length (a store answers 411 to a PUT without one). A negative size is refused, and so
+// is one over limit (MaxDocument when limit is not above zero, ADR 0468 §3), both before any request. A body that
+// yields more or fewer bytes than size fails the request, so a PUT whose body does not end as declared stores nothing.
+func Put(ctx context.Context, client *http.Client, link string, body io.Reader, size, limit int64) (Result, error) {
+	if limit <= 0 {
+		limit = MaxDocument
+	}
+	if client == nil {
+		client = http.DefaultClient
+	}
+	if size < 0 {
+		return Result{}, errors.New("objectlink: put needs the size of the body; a negative or unknown size is refused")
+	}
+	if size > limit {
+		return Result{}, tooBig(limit)
+	}
+	hash := sha256.New()
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	counted := &exact{r: body, size: size, hash: hash}
+	req, err := http.NewRequestWithContext(ctx, http.MethodPut, link, io.NopCloser(counted))
+	if err != nil {
+		return Result{}, err
+	}
+	req.ContentLength = size
+	if size == 0 {
+		req.Body = http.NoBody
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		cancel()
+		return Result{}, fmt.Errorf("objectlink: put: %w", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
+		return Result{}, storeError(resp)
+	}
+	if counted.n != size {
+		return Result{}, fmt.Errorf("objectlink: put: sent %d bytes of the declared %d", counted.n, size)
+	}
+	return Result{Size: counted.n, SHA256: hex.EncodeToString(hash.Sum(nil))}, nil
+}
+
+// Get streams the object of a link made for GET into w, refusing more than limit bytes (MaxDocument when limit is
+// not above zero). When wantSHA256 is given and the bytes hash to another value the call fails: the caller removes
+// what it wrote.
+func Get(ctx context.Context, client *http.Client, link string, w io.Writer, limit int64, wantSHA256 string) (Result, error) {
+	if limit <= 0 {
+		limit = MaxDocument
+	}
+	if client == nil {
+		client = http.DefaultClient
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, link, nil)
+	if err != nil {
+		return Result{}, err
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return Result{}, fmt.Errorf("objectlink: get: %w", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
+		return Result{}, storeError(resp)
+	}
+	if resp.ContentLength > limit {
+		return Result{}, tooBig(limit)
+	}
+	hash := sha256.New()
+	n, err := io.Copy(io.MultiWriter(w, hash), io.LimitReader(resp.Body, limit+1))
+	if err != nil {
+		return Result{}, fmt.Errorf("objectlink: get: %w", err)
+	}
+	if n > limit {
+		return Result{}, tooBig(limit)
+	}
+	got := hex.EncodeToString(hash.Sum(nil))
+	if wantSHA256 != "" && !strings.EqualFold(wantSHA256, got) {
+		return Result{}, fmt.Errorf("objectlink: the document hashes to %s, not the %s expected", got, strings.ToLower(wantSHA256))
+	}
+	return Result{Size: n, SHA256: got}, nil
+}

go/objectlink/objectlink_test.go

@@ -0,0 +1,163 @@
+package objectlink
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"testing"
+	"time"
+)
+
+// Issue 1592: one helper signs and streams a store link (novox/hq ADR 0468 §3).
+
+type roundTrip func(*http.Request) (*http.Response, error)
+
+func (f roundTrip) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
+
+func hexSum(b []byte) string { s := sha256.Sum256(b); return hex.EncodeToString(s[:]) }
+
+// 1592/AC1: AWS's documented query-presign example (a GET of test.txt, 86400 s, 2013-05-24) gives the documented signature.
+func TestPresignMatchesAWSDocumentedExample(t *testing.T) {
+	now := time.Date(2013, 5, 24, 0, 0, 0, 0, time.UTC)
+	query := sign("GET", "examplebucket.s3.amazonaws.com", "/test.txt", "AKIAIOSFODNN7EXAMPLE",
+		"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "us-east-1", now, 86400)
+	const want = "aeeed9bbccd4d02ee5c0109b86d86835f995330da4c265957d157751f604d404"
+	if !strings.HasSuffix(query, "&X-Amz-Signature="+want) {
+		t.Fatalf("the signature differs from AWS's documented one: %s", query)
+	}
+	if !strings.Contains(query, "X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20130524%2Fus-east-1%2Fs3%2Faws4_request") {
+		t.Errorf("the credential is not encoded as documented: %s", query)
+	}
+}
+
+// 1592/AC2: an Expires of 0, below 0 or above 600 s is refused; the signed host is the endpoint's host.
+func TestPresignRefusesBadExpiresAndKeepsTheHost(t *testing.T) {
+	r := Request{Method: "PUT", Endpoint: "http://minio.internal:9000", Bucket: "files", Key: "a b/c.pdf",
+		AccessKey: "ak", SecretKey: "sk", Expires: 10 * time.Minute, Now: time.Unix(1700000000, 0)}
+	link, err := Presign(r)
+	if err != nil {
+		t.Fatal(err)
+	}
+	u, err := url.Parse(link)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if u.Host != "minio.internal:9000" || u.Path != "/files/a b/c.pdf" || u.Query().Get("X-Amz-Expires") != "600" ||
+		u.Query().Get("X-Amz-SignedHeaders") != "host" {
+		t.Errorf("the link is %s", link)
+	}
+	for _, e := range []time.Duration{0, -time.Second, 500 * time.Millisecond, 601 * time.Second, time.Hour} {
+		r.Expires = e
+		if _, err := Presign(r); err == nil {
+			t.Errorf("Expires %s was accepted", e)
+		}
+	}
+}
+
+// 1592/AC3: Put streams and answers size and SHA-256 as the server saw them, always with the declared Content-Length;
+// an unknown size, one past the limit, or a body not ending as declared is refused; a 403 with an S3 error names its code.
+func TestPutStreamsCountsAndRefuses(t *testing.T) {
+	var gotBody []byte
+	var lengths []int64
+	store := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotBody, _ = io.ReadAll(r.Body)
+		lengths = append(lengths, r.ContentLength)
+		if r.URL.Query().Get("deny") != "" {
+			w.WriteHeader(403)
+			io.WriteString(w, `<?xml version="1.0"?><Error><Code>SignatureDoesNotMatch</Code><Message>nope</Message></Error>`)
+		}
+	}))
+	defer store.Close()
+	data := bytes.Repeat([]byte("0123456789"), 100)
+
+	// A reader with no Len or Seek: still sent with the declared length.
+	got, err := Put(context.Background(), store.Client(), store.URL+"/b/k", io.MultiReader(bytes.NewReader(data)), 1000, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got.Size != int64(len(gotBody)) || got.SHA256 != hexSum(gotBody) || got.Size != 1000 {
+		t.Errorf("answered %+v, the server received %d bytes %s", got, len(gotBody), hexSum(gotBody))
+	}
+	if _, err := Put(context.Background(), store.Client(), store.URL+"/b/k", bytes.NewReader(data), 1000, 1000); err != nil {
+		t.Errorf("exactly the limit: %v", err)
+	}
+	for _, l := range lengths {
+		if l != 1000 {
+			t.Errorf("a request carried Content-Length %d, not the declared 1000", l)
+		}
+	}
+	if len(lengths) != 2 {
+		t.Fatalf("the server saw %d requests", len(lengths))
+	}
+
+	// Refused before any request: unknown (negative) size, and a size past the limit.
+	if _, err := Put(context.Background(), store.Client(), store.URL+"/b/k", bytes.NewReader(data), -1, 0); err == nil || !strings.Contains(err.Error(), "size") {
+		t.Errorf("an unknown size: %v", err)
+	}
+	if _, err := Put(context.Background(), store.Client(), store.URL+"/b/k", bytes.NewReader(data), 1000, 999); err == nil || !strings.Contains(err.Error(), "ADR 0468 §3") {
+		t.Errorf("a size past the limit: %v", err)
+	}
+	if len(lengths) != 2 {
+		t.Errorf("a refused call made a request: the server saw %d", len(lengths))
+	}
+
+	// A body longer or shorter than declared fails, and the store is not told it succeeded.
+	reads := &http.Client{Transport: roundTrip(func(r *http.Request) (*http.Response, error) {
+		_, err := io.Copy(io.Discard, r.Body)
+		if err == nil {
+			return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(""))}, nil
+		}
+		return nil, err
+	})}
+	for name, declared := range map[string]int64{"longer than declared": 999, "shorter than declared": 1001} {
+		if _, err := Put(context.Background(), reads, store.URL+"/b/k", bytes.NewReader(data), declared, 0); err == nil {
+			t.Errorf("a body %s was accepted", name)
+		}
+	}
+
+	_, err = Put(context.Background(), store.Client(), store.URL+"/b/k?deny=1", bytes.NewReader(data), 1000, 0)
+	e, ok := err.(*StoreError)
+	if !ok || e.Status != 403 || e.Code != "SignatureDoesNotMatch" || !strings.Contains(e.Error(), "SignatureDoesNotMatch") {
+		t.Fatalf("the refusal is %v", err)
+	}
+}
+
+// 1592/AC4: Get writes the body and answers size and SHA-256; a wrong digest and a document past the limit are errors.
+func TestGetStreamsChecksAndRefuses(t *testing.T) {
+	data := bytes.Repeat([]byte("abcdefghij"), 100)
+	store := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write(data)
+	}))
+	defer store.Close()
+	var out bytes.Buffer
+	got, err := Get(context.Background(), store.Client(), store.URL+"/b/k", &out, 0, hexSum(data))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got.Size != 1000 || got.SHA256 != hexSum(data) || !bytes.Equal(out.Bytes(), data) {
+		t.Errorf("answered %+v, wrote %d bytes", got, out.Len())
+	}
+	if _, err := Get(context.Background(), store.Client(), store.URL+"/b/k", io.Discard, 0, hexSum([]byte("other"))); err == nil {
+		t.Error("a wrong digest was accepted")
+	}
+	if _, err := Get(context.Background(), store.Client(), store.URL+"/b/k", io.Discard, 999, ""); err == nil {
+		t.Error("a document past the limit was accepted")
+	}
+	chunked := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.(http.Flusher).Flush()
+		w.Write(data)
+	}))
+	defer chunked.Close()
+	if _, err := Get(context.Background(), chunked.Client(), chunked.URL+"/b/k", io.Discard, 999, ""); err == nil {
+		t.Error("a chunked document past the limit was accepted")
+	}
+	if _, err := Get(context.Background(), chunked.Client(), chunked.URL+"/b/k", io.Discard, 1000, ""); err != nil {
+		t.Errorf("exactly the limit: %v", err)
+	}
+}