diff --git a/go/catalogue/bucket_shares.go b/go/catalogue/bucket_shares.go new file mode 100644 index 0000000..9443f7c --- /dev/null +++ b/go/catalogue/bucket_shares.go @@ -0,0 +1,124 @@ +package catalogue + +import ( + "encoding/json" + "fmt" + "regexp" + "sort" +) + +// BucketShare is one entry of `shares` on a module's requirement of `s3-bucket` (novox/hq ADR 0468 §8): who else may +// put or get in the consumer's bucket, and with which methods. Not the NFS shares of shares.go (ADR 0263). +// +// "requires": [{"name": "s3-bucket", "shares": [{"to": "seat:node-files", "methods": ["put"]}, +// {"to": "print-and-scan", "methods": ["get"]}]}] +// +// `to` is a module name, or `seat:`. `methods` is a non-empty subset of get and put, without a repeat. +// Shares are allowed on `s3-bucket` only, and no two entries name the same `to`. +type BucketShare struct { + To string `json:"to"` + Methods []string `json:"methods"` +} + +// BucketRequirement is the requirement `shares` may be written on. +const BucketRequirement = "s3-bucket" + +// RequireSharesKey is where a stored manifest keeps the shares of its requirements: beside `requires`, which stays a +// plain list of names, for the reason OptionalRequiresKey is beside it. An older controller sets the unknown key +// aside instead of failing the catalogue. +const RequireSharesKey = "requires-shares" + +// SharesValue is the key of the value a consumer gives the provider of s3-bucket in its grant: the list of +// {to, methods} sorted by `to`. A consumer's own `contributes` may not name it. +const SharesValue = "shares" + +var shareTarget = regexp.MustCompile(`^(seat:)?[a-z0-9][a-z0-9-]*$`) + +// checkBucketShares refuses a malformed `shares` of one requirement, naming the entry and ADR 0468 §8. +func checkBucketShares(requirement string, shares []BucketShare) error { + if len(shares) == 0 { + return nil + } + if requirement != BucketRequirement { + return fmt.Errorf("requires %s: `shares` is allowed on %s only (novox/hq ADR 0468 §8)", requirement, BucketRequirement) + } + seen := map[string]bool{} + for i, s := range shares { + at := fmt.Sprintf("requires %s: shares[%d] (to %q)", requirement, i, s.To) + if !shareTarget.MatchString(s.To) { + return fmt.Errorf("%s: `to` is a module name or seat: (novox/hq ADR 0468 §8)", at) + } + if seen[s.To] { + return fmt.Errorf("%s: a second entry for the same `to` (novox/hq ADR 0468 §8)", at) + } + seen[s.To] = true + if len(s.Methods) == 0 { + return fmt.Errorf("%s: `methods` is empty; name get, put or both (novox/hq ADR 0468 §8)", at) + } + used := map[string]bool{} + for _, method := range s.Methods { + if method != "get" && method != "put" { + return fmt.Errorf("%s: method %q is not get or put (novox/hq ADR 0468 §8)", at, method) + } + if used[method] { + return fmt.Errorf("%s: method %q twice (novox/hq ADR 0468 §8)", at, method) + } + used[method] = true + } + } + return nil +} + +// readRequireShares reads the stored form's side key. +func readRequireShares(raw json.RawMessage) (map[string][]BucketShare, error) { + var out map[string][]BucketShare + if err := json.Unmarshal(raw, &out); err != nil { + return nil, fmt.Errorf("%s: an object of requirement to a list of {to, methods}: %w", RequireSharesKey, err) + } + for requirement, shares := range out { + if err := checkBucketShares(requirement, shares); err != nil { + return nil, err + } + } + return out, nil +} + +// sortedShares is a requirement's shares as they are handed to the provider: sorted by `to` (a file whose lines move +// looks changed), each entry's methods sorted, and a copy so the manifest is not reordered. +func sortedShares(shares []BucketShare) []BucketShare { + out := make([]BucketShare, 0, len(shares)) + for _, s := range shares { + methods := append([]string(nil), s.Methods...) + sort.Strings(methods) + out = append(out, BucketShare{To: s.To, Methods: methods}) + } + sort.Slice(out, func(i, j int) bool { return out[i].To < out[j].To }) + return out +} + +// withShares adds this module's shares of the requirement to the values it gives the provider. A module whose own +// `contributes` already names `shares` is refused: the field is the only way to say it (ADR 0468 §8). +func withShares(m Manifest, requirement string, values map[string]any) (map[string]any, error) { + shares := m.RequireShares[requirement] + if _, named := values[SharesValue]; named && requirement == BucketRequirement { + return nil, fmt.Errorf("%s contributing to %s names `shares`; write it on its requirement instead (novox/hq ADR 0468 §8)", + m.Module, requirement) + } + if len(shares) == 0 { + return values, nil + } + out := make(map[string]any, len(values)+1) + for k, v := range values { + out[k] = v + } + listed := []any{} + for _, s := range sortedShares(shares) { + methods := []any{} + for _, method := range s.Methods { + methods = append(methods, method) + } + listed = append(listed, map[string]any{"to": s.To, "methods": methods}) + } + out[SharesValue] = listed + return out, nil +} diff --git a/go/catalogue/bucket_shares_test.go b/go/catalogue/bucket_shares_test.go new file mode 100644 index 0000000..87345f4 --- /dev/null +++ b/go/catalogue/bucket_shares_test.go @@ -0,0 +1,185 @@ +package catalogue + +import ( + "encoding/json" + "reflect" + "strings" + "testing" +) + +// Issue 1591: `shares` on a requirement of s3-bucket (novox/hq ADR 0468 §8). + +const filesWithShares = `{"module":"files","version":"1","requires":["shell", + {"name":"s3-bucket","shares":[{"to":"seat:node-files","methods":["put"]},{"to":"print-and-scan","methods":["get","put"]}]}]}` + +func minioLike() Manifest { + return Manifest{Module: "minio", Version: "1", Provides: Offers("s3-bucket"), + Receives: map[string]string{"s3-bucket": "/grants/mesh.json"}} +} + +func givenToMinio(t *testing.T, out []map[string]any) []Contribution { + t.Helper() + for _, r := range out { + if r["path"] != "/grants/mesh.json" { + continue + } + var parsed struct { + Given []Contribution `json:"given"` + } + if err := json.Unmarshal([]byte(r["content"].(string)), &parsed); err != nil { + t.Fatal(err) + } + return parsed.Given + } + t.Fatalf("minio was given no file: %v", out) + return nil +} + +func sharesOf(t *testing.T, values map[string]any) string { + t.Helper() + raw, err := json.Marshal(values["shares"]) + if err != nil { + t.Fatal(err) + } + return string(raw) +} + +const sortedSharesJSON = `[{"methods":["get","put"],"to":"print-and-scan"},{"methods":["put"],"to":"seat:node-files"}]` + +// 1591/AC1: parsed, the shares are on the manifest; stored, `requires` is plain names and the side key holds them. +func TestSharesParseAndAreStoredBesideRequires(t *testing.T) { + m := mustParse(t, filesWithShares) + want := []BucketShare{{To: "seat:node-files", Methods: []string{"put"}}, {To: "print-and-scan", Methods: []string{"get", "put"}}} + if !reflect.DeepEqual(m.RequireShares["s3-bucket"], want) { + t.Fatalf("the shares read as %+v", m.RequireShares) + } + raw, err := json.Marshal(m) + if err != nil { + t.Fatal(err) + } + var keys map[string]json.RawMessage + if err := json.Unmarshal(raw, &keys); err != nil { + t.Fatal(err) + } + if string(keys["requires"]) != `["shell","s3-bucket"]` { + t.Errorf("the stored requires is not plain names: %s", keys["requires"]) + } + if !strings.Contains(string(keys[RequireSharesKey]), `"seat:node-files"`) { + t.Errorf("the side key does not hold the shares: %s", raw) + } + var again Manifest + if err := json.Unmarshal(raw, &again); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(again, m) { + t.Fatalf("not lossless:\n%+v\n%+v", again, m) + } +} + +// 1591/AC2: a malformed share is refused, naming the entry and ADR 0468 §8. +func TestAMalformedShareIsRefused(t *testing.T) { + cases := map[string]string{ + "unknown method": `{"name":"s3-bucket","shares":[{"to":"a","methods":["delete"]}]}`, + "empty to": `{"name":"s3-bucket","shares":[{"to":"","methods":["get"]}]}`, + "malformed to": `{"name":"s3-bucket","shares":[{"to":"Seat:X y","methods":["get"]}]}`, + "empty methods": `{"name":"s3-bucket","shares":[{"to":"a","methods":[]}]}`, + "duplicate to": `{"name":"s3-bucket","shares":[{"to":"a","methods":["get"]},{"to":"a","methods":["put"]}]}`, + "duplicate verb": `{"name":"s3-bucket","shares":[{"to":"a","methods":["get","get"]}]}`, + "other require": `{"name":"route","shares":[{"to":"a","methods":["get"]}]}`, + } + for name, entry := range cases { + _, err := ParseManifest([]byte(`{"module":"files","version":"1","requires":[` + entry + `]}`)) + if err == nil { + t.Errorf("%s: accepted", name) + continue + } + if !strings.Contains(err.Error(), "ADR 0468 §8") || !strings.Contains(err.Error(), "requires") { + t.Errorf("%s: the error does not name the entry and the ADR: %v", name, err) + } + } + // The stored form is held to the same rules. + _, err := ParseManifest([]byte(`{"module":"files","version":"1","requires":["s3-bucket"], + "requires-shares":{"s3-bucket":[{"to":"a","methods":[]}]}}`)) + if err == nil || !strings.Contains(err.Error(), "ADR 0468 §8") { + t.Errorf("the stored form was accepted: %v", err) + } +} + +// 1591/AC3: the values a consumer gives carry the shares sorted by `to`, remote (grant) and co-located alike. +func TestTheGrantCarriesTheSharesSorted(t *testing.T) { + m := mustParse(t, filesWithShares) + remote := Resolution{Node: "laptop", Modules: []Manifest{m}} + values, asks, err := remote.ContributionsFrom("s3-bucket", "files", SettingsBy{}) + if err != nil || !asks { + t.Fatalf("asks %v, err %v", asks, err) + } + if got := sharesOf(t, values); got != sortedSharesJSON { + t.Fatalf("the grant values carry %s", got) + } + // As the provider's file shows it, through a grant. + provider, err := Resolve(shelf(minioLike()), []string{"minio"}, workstation(), World{}) + if err != nil { + t.Fatal(err) + } + out, err := provider.Declaration(Rendering{Grants: []Grant{{Provision: "s3-bucket", Consumer: "laptop", From: "files", Values: values}}}) + if err != nil { + t.Fatal(err) + } + given := givenToMinio(t, out) + if len(given) != 1 || sharesOf(t, given[0].Values) != sortedSharesJSON { + t.Fatalf("the remote consumer's shares did not reach the file: %+v", given) + } + + // Co-located: the consumer contributes to s3-bucket too, and no grant is passed. + co := mustParse(t, `{"module":"files","version":"1","contributes":{"s3-bucket":{"name":"files"}}, + "requires":[{"name":"s3-bucket","shares":[{"to":"seat:node-files","methods":["put"]},{"to":"print-and-scan","methods":["put","get"]}]}]}`) + both, err := Resolve(shelf(minioLike(), co), []string{"files"}, workstation(), World{}) + if err != nil { + t.Fatal(err) + } + out, err = both.Declaration(Rendering{}) + if err != nil { + t.Fatal(err) + } + given = givenToMinio(t, out) + if len(given) != 1 || sharesOf(t, given[0].Values) != sortedSharesJSON || given[0].Values["name"] != "files" { + t.Fatalf("the co-located consumer's shares did not reach the file: %+v", given) + } +} + +// 1591/AC4: only the shares change, so the module is not recreated and held for no one; the provider's file changes. +func TestChangingOnlyTheSharesDoesNotRecreateTheConsumer(t *testing.T) { + was := mustParse(t, filesWithShares) + now := mustParse(t, `{"module":"files","version":"1","requires":["shell", + {"name":"s3-bucket","shares":[{"to":"print-and-scan","methods":["get"]}]}]}`) + if said := Recreates(was, now).Say("files"); said != "" { + t.Errorf("a change of shares recreates the module: %s", said) + } + if said := Holds(was, now); said != "" { + t.Errorf("a change of shares holds the module: %s", said) + } + before := Resolution{Node: "laptop", Modules: []Manifest{was}} + after := Resolution{Node: "laptop", Modules: []Manifest{now}} + a, _, _ := before.ContributionsFrom("s3-bucket", "files", SettingsBy{}) + b, _, _ := after.ContributionsFrom("s3-bucket", "files", SettingsBy{}) + if sharesOf(t, a) == sharesOf(t, b) { + t.Fatalf("the provider's values did not change: %s", sharesOf(t, a)) + } +} + +// 1591/AC5: a consumer whose contributes to s3-bucket names `shares` itself is refused. +func TestAConsumerCannotContributeSharesItself(t *testing.T) { + _, err := ParseManifest([]byte(`{"module":"files","version":"1", + "contributes":{"s3-bucket":{"shares":"x"}}, + "requires":[{"name":"s3-bucket","shares":[{"to":"a","methods":["get"]}]}]}`)) + if err == nil || !strings.Contains(err.Error(), "ADR 0468 §8") { + t.Errorf("both written, accepted: %v", err) + } + // Without the field the grant is still refused, so the provider is never told something the manifest did not say. + m := Manifest{Module: "files", Requires: []string{"s3-bucket"}, + Contributes: map[string]map[string]any{"s3-bucket": {"shares": "x"}}} + r := Resolution{Node: "laptop", Modules: []Manifest{m}} + if _, _, err := r.ContributionsFrom("s3-bucket", "files", SettingsBy{}); err == nil { + t.Error("a contributed `shares` reached the grant") + } +} diff --git a/go/catalogue/declaration.go b/go/catalogue/declaration.go index a2d7597..35895c8 100644 --- a/go/catalogue/declaration.go +++ b/go/catalogue/declaration.go @@ -1783,6 +1783,10 @@ func (r Resolution) contributions(settings SettingsBy, grants []Grant, if err != nil { return nil, err } + // What the module shares of its bucket reaches the provider with the rest (novox/hq ADR 0468 §8). + if values, err = withShares(m, to, values); err != nil { + return nil, err + } out[to] = append(out[to], Contribution{From: m.Module, Values: values}) } // Several contributions to one requirement (ADR 0094's sibling for `contributes`): an @@ -2169,11 +2173,19 @@ func (r Resolution) ContributionsFrom(requirement, module string, settings Setti mine = append(mine, g.Values) } } + var asker Manifest + for _, m := range r.Modules { + if m.Module == module { + asker = m + } + } if len(mine) == 1 { + // Its shares are already in, put there where its contribution was composed (contributions). return mine[0], true, nil } if len(mine) > 1 { - return map[string]any{}, true, nil + values, err := withShares(asker, requirement, map[string]any{}) + return values, err == nil, err } // It contributes no payload — but a require-only consumer of a parameterless provision (one whose // `serves` names no consumer-supplied key: `redis-cache`, `amqp`) still ASKS for it and must be @@ -2186,7 +2198,8 @@ func (r Resolution) ContributionsFrom(requirement, module string, settings Setti } for _, req := range m.Requires { if req == requirement { - return map[string]any{}, true, nil + values, err := withShares(m, requirement, map[string]any{}) + return values, err == nil, err } } } diff --git a/go/catalogue/manifest.go b/go/catalogue/manifest.go index cd48fdf..0464654 100644 --- a/go/catalogue/manifest.go +++ b/go/catalogue/manifest.go @@ -494,6 +494,9 @@ type Manifest struct { // by UnmarshalJSON from the long form of a `requires` entry, {name, optional}, or from the stored // form's `optional-requires` list; never written by hand. OptionalRequires map[string]bool `json:"-"` + // RequireShares is who else may use the bucket a module requires, by requirement (BucketShare, novox/hq ADR 0468 + // §8): filled from the long form's `shares`, or from the stored form's `requires-shares`; never written by hand. + RequireShares map[string][]BucketShare `json:"-"` // Emits are the events this module publishes, named **locally**: `order.placed`, not a subject // and not a routing key. The mesh derives where it lands (design 29 §1), so reorganising the @@ -1538,8 +1541,9 @@ func (m *Manifest) UnmarshalJSON(raw []byte) error { delete(keys, "contributes") } var optionalRequires map[string]bool + var requireShares map[string][]BucketShare if requires, ok := keys["requires"]; ok && string(requires) != "null" { - names, optional, err := readRequires(requires) + names, optional, shares, err := readRequires(requires) if err != nil { return err } @@ -1549,6 +1553,23 @@ func (m *Manifest) UnmarshalJSON(raw []byte) error { } keys["requires"] = plainNames optionalRequires = optional + requireShares = shares + } + // The stored form's shares (RequireSharesKey). + if stored, ok := keys[RequireSharesKey]; ok { + if string(stored) != "null" { + stood, err := readRequireShares(stored) + if err != nil { + return err + } + for requirement, shares := range stood { + if requireShares == nil { + requireShares = map[string][]BucketShare{} + } + requireShares[requirement] = shares + } + } + delete(keys, RequireSharesKey) } // The stored form: `requires` plain, and which are optional beside it (OptionalRequiresKey). if stored, ok := keys[OptionalRequiresKey]; ok { @@ -1599,6 +1620,7 @@ func (m *Manifest) UnmarshalJSON(raw []byte) error { m.unknown = unknown m.forms = forms m.OptionalRequires = optionalRequires + m.RequireShares = requireShares if len(plain) > 0 { m.Secrets = plain } @@ -1611,6 +1633,21 @@ func (m *Manifest) UnmarshalJSON(raw []byte) error { if len(contributesMany) > 0 { m.ContributesMany = contributesMany } + if err := m.checkSharesNotContributed(); err != nil { + return err + } + return nil +} + +// checkSharesNotContributed refuses a module whose own `contributes` to s3-bucket names `shares` (ADR 0468 §8). +func (m Manifest) checkSharesNotContributed() error { + if len(m.RequireShares[BucketRequirement]) == 0 { + return nil + } + if _, named := m.Contributes[BucketRequirement][SharesValue]; named { + return fmt.Errorf("contributes.%s names `shares` and so does its requirement; keep one, the requirement's (novox/hq ADR 0468 §8)", + BucketRequirement) + } return nil } @@ -1655,7 +1692,7 @@ func (m Manifest) marshalFields() ([]byte, error) { if err != nil { return nil, err } - if len(m.SecretsMany) == 0 && len(m.ContributesMany) == 0 && len(m.OptionalRequires) == 0 { + if len(m.SecretsMany) == 0 && len(m.ContributesMany) == 0 && len(m.OptionalRequires) == 0 && len(m.RequireShares) == 0 { return raw, nil } var keys map[string]json.RawMessage @@ -1671,6 +1708,13 @@ func (m Manifest) marshalFields() ([]byte, error) { } keys[OptionalRequiresKey] = listed } + if len(m.RequireShares) > 0 { + listed, err := json.Marshal(m.RequireShares) + if err != nil { + return nil, err + } + keys[RequireSharesKey] = listed + } if len(m.ContributesMany) > 0 { mergedContributes := map[string]any{} for to, values := range m.Contributes { diff --git a/go/catalogue/optional_requirements.go b/go/catalogue/optional_requirements.go index f2db8ad..921b1d7 100644 --- a/go/catalogue/optional_requirements.go +++ b/go/catalogue/optional_requirements.go @@ -32,17 +32,20 @@ const OnlyWhenBound = "only-when-bound" type requirementForm struct { Name string `json:"name"` Optional bool `json:"optional,omitempty"` + // Shares is who else may use the consumer's bucket (BucketShare, novox/hq ADR 0468 §8), on s3-bucket only. + Shares []BucketShare `json:"shares,omitempty"` } // readRequires reads `requires` in both of its forms — a bare name, or {name, optional} — into the // names and which of them are optional. -func readRequires(raw json.RawMessage) ([]string, map[string]bool, error) { +func readRequires(raw json.RawMessage) ([]string, map[string]bool, map[string][]BucketShare, error) { var entries []json.RawMessage if err := json.Unmarshal(raw, &entries); err != nil { - return nil, nil, fmt.Errorf("requires: a list of names, or of {name, optional}: %w", err) + return nil, nil, nil, fmt.Errorf("requires: a list of names, or of {name, optional}: %w", err) } var names []string optional := map[string]bool{} + var shares map[string][]BucketShare for i, e := range entries { var plain string if err := json.Unmarshal(e, &plain); err == nil { @@ -53,12 +56,21 @@ func readRequires(raw json.RawMessage) ([]string, map[string]bool, error) { dec := json.NewDecoder(bytes.NewReader(e)) dec.DisallowUnknownFields() if err := dec.Decode(&full); err != nil { - return nil, nil, fmt.Errorf("requires[%d]: a name, or {name, optional} (novox/hq ADR 0330): %w", i, err) + return nil, nil, nil, fmt.Errorf("requires[%d]: a name, or {name, optional} (novox/hq ADR 0330): %w", i, err) } if strings.TrimSpace(full.Name) == "" { - return nil, nil, fmt.Errorf("requires[%d] names no requirement: {name, optional}", i) + return nil, nil, nil, fmt.Errorf("requires[%d] names no requirement: {name, optional}", i) } names = append(names, full.Name) + if err := checkBucketShares(full.Name, full.Shares); err != nil { + return nil, nil, nil, fmt.Errorf("requires[%d]: %w", i, err) + } + if len(full.Shares) > 0 { + if shares == nil { + shares = map[string][]BucketShare{} + } + shares[full.Name] = full.Shares + } if full.Optional { optional[full.Name] = true } @@ -66,7 +78,7 @@ func readRequires(raw json.RawMessage) ([]string, map[string]bool, error) { if len(optional) == 0 { optional = nil } - return names, optional, nil + return names, optional, shares, nil } // OptionalRequiresKey is where a stored manifest keeps which of its requirements are optional: beside diff --git a/go/objectlink/objectlink.go b/go/objectlink/objectlink.go new file mode 100644 index 0000000..5e58048 --- /dev/null +++ b/go/objectlink/objectlink.go @@ -0,0 +1,277 @@ +// Package objectlink signs and streams a link to one object in the mesh's object store (novox/hq ADR 0468 §3). +// +// Bytes move over HTTP between a party and the store, on a signed link of at most ten minutes for one object and one +// method, signed for the name the `s3-bucket` binding gives; a document over 100 MiB is refused. Presign makes the +// link (Signature Version 4, query form, path style, payload unsigned, `host` the only signed header); Put and Get +// move the bytes without holding them in memory and count and hash them as they go. +// +// Standard library only: the SDK requires nothing of the mesh's forge. +package objectlink + +import ( + "context" + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "encoding/xml" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strings" + "time" +) + +const ( + // MaxDocument is the largest document a link moves (ADR 0468 §3): the default limit of Put and Get. + MaxDocument int64 = 100 << 20 + // MaxExpires is the longest a link lives (ADR 0468 §3). + MaxExpires = 10 * time.Minute + + defaultRegion = "us-east-1" + algorithm = "AWS4-HMAC-SHA256" +) + +// Request is one link to sign. +type Request struct { + // Method is GET or PUT. + Method string + // Endpoint is the store as the party reaches it, "http://host:port". Its host is signed exactly as written. + Endpoint string + Bucket string + Key string + // AccessKey and SecretKey are the credential the s3-bucket binding gives. + AccessKey string + SecretKey string + // Region defaults to us-east-1. + Region string + // Expires is how long the link lives: at least one second, at most ten minutes. + Expires time.Duration + // Now is the signing time; the clock's when zero. + Now time.Time +} + +// Presign returns the signed link for one object and one method. +func Presign(r Request) (string, error) { + if r.Method != http.MethodGet && r.Method != http.MethodPut { + return "", fmt.Errorf("objectlink: method %q: a link is for GET or PUT (ADR 0468 §3)", r.Method) + } + if r.Expires < time.Second || r.Expires > MaxExpires { + return "", fmt.Errorf("objectlink: a link lives at least 1s and at most %s, not %s (ADR 0468 §3)", MaxExpires, r.Expires) + } + if r.Bucket == "" || r.Key == "" || r.AccessKey == "" || r.SecretKey == "" { + return "", errors.New("objectlink: a bucket, a key and a credential are needed to sign a link") + } + u, err := url.Parse(r.Endpoint) + if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") { + return "", fmt.Errorf("objectlink: endpoint %q is not http(s)://host[:port]", r.Endpoint) + } + region := r.Region + if region == "" { + region = defaultRegion + } + now := r.Now + if now.IsZero() { + now = time.Now() + } + path := "/" + encode(r.Bucket, false) + "/" + encode(r.Key, false) + query := sign(r.Method, u.Host, path, r.AccessKey, r.SecretKey, region, now.UTC(), int64(r.Expires/time.Second)) + return u.Scheme + "://" + u.Host + path + "?" + query, nil +} + +// sign returns the query string of a presigned link, signature last. The path is already encoded. +func sign(method, host, path, accessKey, secretKey, region string, now time.Time, expires int64) string { + date := now.Format("20060102") + stamp := now.Format("20060102T150405Z") + scope := date + "/" + region + "/s3/aws4_request" + params := map[string]string{ + "X-Amz-Algorithm": algorithm, + "X-Amz-Credential": accessKey + "/" + scope, + "X-Amz-Date": stamp, + "X-Amz-Expires": fmt.Sprint(expires), + "X-Amz-SignedHeaders": "host", + } + names := make([]string, 0, len(params)) + for k := range params { + names = append(names, k) + } + sort.Strings(names) + var query []string + for _, k := range names { + query = append(query, encode(k, true)+"="+encode(params[k], true)) + } + canonicalQuery := strings.Join(query, "&") + canonical := strings.Join([]string{method, path, canonicalQuery, "host:" + host + "\n", "host", "UNSIGNED-PAYLOAD"}, "\n") + digest := sha256.Sum256([]byte(canonical)) + toSign := strings.Join([]string{algorithm, stamp, scope, hex.EncodeToString(digest[:])}, "\n") + key := mac([]byte("AWS4"+secretKey), date) + key = mac(key, region) + key = mac(key, "s3") + key = mac(key, "aws4_request") + return canonicalQuery + "&X-Amz-Signature=" + hex.EncodeToString(mac(key, toSign)) +} + +func mac(key []byte, data string) []byte { + h := hmac.New(sha256.New, key) + h.Write([]byte(data)) + return h.Sum(nil) +} + +// encode is AWS's URI encoding: everything but A-Z a-z 0-9 - _ . ~ as %XX, and the slash kept in a path. +func encode(s string, slash bool) string { + var b strings.Builder + for i := 0; i < len(s); i++ { + c := s[i] + switch { + case c >= 'A' && c <= 'Z', c >= 'a' && c <= 'z', c >= '0' && c <= '9', c == '-', c == '_', c == '.', c == '~': + b.WriteByte(c) + case c == '/' && !slash: + b.WriteByte(c) + default: + fmt.Fprintf(&b, "%%%02X", c) + } + } + return b.String() +} + +// Result is what moved: its size and the SHA-256 of its bytes, in hex. +type Result struct { + Size int64 + SHA256 string +} + +// StoreError is a refusal by the store: its status and the code and message of its XML error body. +type StoreError struct { + Status int + Code string + Message string +} + +func (e *StoreError) Error() string { + if e.Code == "" { + return fmt.Sprintf("objectlink: the store answered %d", e.Status) + } + return fmt.Sprintf("objectlink: the store answered %d %s: %s", e.Status, e.Code, e.Message) +} + +func storeError(resp *http.Response) error { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<10)) + var parsed struct { + Code string `xml:"Code"` + Message string `xml:"Message"` + } + _ = xml.Unmarshal(body, &parsed) + return &StoreError{Status: resp.StatusCode, Code: parsed.Code, Message: parsed.Message} +} + +// tooBig is the error of a document past its limit. +func tooBig(limit int64) error { + return fmt.Errorf("objectlink: the document is over %d bytes (ADR 0468 §3)", limit) +} + +// exact counts and hashes what passes, and fails a stream that is longer than declared, or ends before it. +type exact struct { + r io.Reader + size int64 + n int64 + hash interface{ Write([]byte) (int, error) } +} + +func (l *exact) Read(p []byte) (int, error) { + n, err := l.r.Read(p) + l.n += int64(n) + if l.n > l.size { + return 0, fmt.Errorf("objectlink: the body is longer than the declared %d bytes", l.size) + } + l.hash.Write(p[:n]) + if err == io.EOF && l.n < l.size { + return n, fmt.Errorf("objectlink: the body ended at %d bytes, short of the declared %d", l.n, l.size) + } + return n, err +} + +// Put streams body to a link made for PUT, never holding it in memory. The caller gives the body's exact size, which +// is always sent as the Content-Length (a store answers 411 to a PUT without one). A negative size is refused, and so +// is one over limit (MaxDocument when limit is not above zero, ADR 0468 §3), both before any request. A body that +// yields more or fewer bytes than size fails the request, so a PUT whose body does not end as declared stores nothing. +func Put(ctx context.Context, client *http.Client, link string, body io.Reader, size, limit int64) (Result, error) { + if limit <= 0 { + limit = MaxDocument + } + if client == nil { + client = http.DefaultClient + } + if size < 0 { + return Result{}, errors.New("objectlink: put needs the size of the body; a negative or unknown size is refused") + } + if size > limit { + return Result{}, tooBig(limit) + } + hash := sha256.New() + ctx, cancel := context.WithCancel(ctx) + defer cancel() + counted := &exact{r: body, size: size, hash: hash} + req, err := http.NewRequestWithContext(ctx, http.MethodPut, link, io.NopCloser(counted)) + if err != nil { + return Result{}, err + } + req.ContentLength = size + if size == 0 { + req.Body = http.NoBody + } + resp, err := client.Do(req) + if err != nil { + cancel() + return Result{}, fmt.Errorf("objectlink: put: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode > 299 { + return Result{}, storeError(resp) + } + if counted.n != size { + return Result{}, fmt.Errorf("objectlink: put: sent %d bytes of the declared %d", counted.n, size) + } + return Result{Size: counted.n, SHA256: hex.EncodeToString(hash.Sum(nil))}, nil +} + +// Get streams the object of a link made for GET into w, refusing more than limit bytes (MaxDocument when limit is +// not above zero). When wantSHA256 is given and the bytes hash to another value the call fails: the caller removes +// what it wrote. +func Get(ctx context.Context, client *http.Client, link string, w io.Writer, limit int64, wantSHA256 string) (Result, error) { + if limit <= 0 { + limit = MaxDocument + } + if client == nil { + client = http.DefaultClient + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, link, nil) + if err != nil { + return Result{}, err + } + resp, err := client.Do(req) + if err != nil { + return Result{}, fmt.Errorf("objectlink: get: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode > 299 { + return Result{}, storeError(resp) + } + if resp.ContentLength > limit { + return Result{}, tooBig(limit) + } + hash := sha256.New() + n, err := io.Copy(io.MultiWriter(w, hash), io.LimitReader(resp.Body, limit+1)) + if err != nil { + return Result{}, fmt.Errorf("objectlink: get: %w", err) + } + if n > limit { + return Result{}, tooBig(limit) + } + got := hex.EncodeToString(hash.Sum(nil)) + if wantSHA256 != "" && !strings.EqualFold(wantSHA256, got) { + return Result{}, fmt.Errorf("objectlink: the document hashes to %s, not the %s expected", got, strings.ToLower(wantSHA256)) + } + return Result{Size: n, SHA256: got}, nil +} diff --git a/go/objectlink/objectlink_test.go b/go/objectlink/objectlink_test.go new file mode 100644 index 0000000..ae994e0 --- /dev/null +++ b/go/objectlink/objectlink_test.go @@ -0,0 +1,163 @@ +package objectlink + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "io" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" +) + +// Issue 1592: one helper signs and streams a store link (novox/hq ADR 0468 §3). + +type roundTrip func(*http.Request) (*http.Response, error) + +func (f roundTrip) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +func hexSum(b []byte) string { s := sha256.Sum256(b); return hex.EncodeToString(s[:]) } + +// 1592/AC1: AWS's documented query-presign example (a GET of test.txt, 86400 s, 2013-05-24) gives the documented signature. +func TestPresignMatchesAWSDocumentedExample(t *testing.T) { + now := time.Date(2013, 5, 24, 0, 0, 0, 0, time.UTC) + query := sign("GET", "examplebucket.s3.amazonaws.com", "/test.txt", "AKIAIOSFODNN7EXAMPLE", + "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "us-east-1", now, 86400) + const want = "aeeed9bbccd4d02ee5c0109b86d86835f995330da4c265957d157751f604d404" + if !strings.HasSuffix(query, "&X-Amz-Signature="+want) { + t.Fatalf("the signature differs from AWS's documented one: %s", query) + } + if !strings.Contains(query, "X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20130524%2Fus-east-1%2Fs3%2Faws4_request") { + t.Errorf("the credential is not encoded as documented: %s", query) + } +} + +// 1592/AC2: an Expires of 0, below 0 or above 600 s is refused; the signed host is the endpoint's host. +func TestPresignRefusesBadExpiresAndKeepsTheHost(t *testing.T) { + r := Request{Method: "PUT", Endpoint: "http://minio.internal:9000", Bucket: "files", Key: "a b/c.pdf", + AccessKey: "ak", SecretKey: "sk", Expires: 10 * time.Minute, Now: time.Unix(1700000000, 0)} + link, err := Presign(r) + if err != nil { + t.Fatal(err) + } + u, err := url.Parse(link) + if err != nil { + t.Fatal(err) + } + if u.Host != "minio.internal:9000" || u.Path != "/files/a b/c.pdf" || u.Query().Get("X-Amz-Expires") != "600" || + u.Query().Get("X-Amz-SignedHeaders") != "host" { + t.Errorf("the link is %s", link) + } + for _, e := range []time.Duration{0, -time.Second, 500 * time.Millisecond, 601 * time.Second, time.Hour} { + r.Expires = e + if _, err := Presign(r); err == nil { + t.Errorf("Expires %s was accepted", e) + } + } +} + +// 1592/AC3: Put streams and answers size and SHA-256 as the server saw them, always with the declared Content-Length; +// an unknown size, one past the limit, or a body not ending as declared is refused; a 403 with an S3 error names its code. +func TestPutStreamsCountsAndRefuses(t *testing.T) { + var gotBody []byte + var lengths []int64 + store := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotBody, _ = io.ReadAll(r.Body) + lengths = append(lengths, r.ContentLength) + if r.URL.Query().Get("deny") != "" { + w.WriteHeader(403) + io.WriteString(w, `SignatureDoesNotMatchnope`) + } + })) + defer store.Close() + data := bytes.Repeat([]byte("0123456789"), 100) + + // A reader with no Len or Seek: still sent with the declared length. + got, err := Put(context.Background(), store.Client(), store.URL+"/b/k", io.MultiReader(bytes.NewReader(data)), 1000, 0) + if err != nil { + t.Fatal(err) + } + if got.Size != int64(len(gotBody)) || got.SHA256 != hexSum(gotBody) || got.Size != 1000 { + t.Errorf("answered %+v, the server received %d bytes %s", got, len(gotBody), hexSum(gotBody)) + } + if _, err := Put(context.Background(), store.Client(), store.URL+"/b/k", bytes.NewReader(data), 1000, 1000); err != nil { + t.Errorf("exactly the limit: %v", err) + } + for _, l := range lengths { + if l != 1000 { + t.Errorf("a request carried Content-Length %d, not the declared 1000", l) + } + } + if len(lengths) != 2 { + t.Fatalf("the server saw %d requests", len(lengths)) + } + + // Refused before any request: unknown (negative) size, and a size past the limit. + if _, err := Put(context.Background(), store.Client(), store.URL+"/b/k", bytes.NewReader(data), -1, 0); err == nil || !strings.Contains(err.Error(), "size") { + t.Errorf("an unknown size: %v", err) + } + if _, err := Put(context.Background(), store.Client(), store.URL+"/b/k", bytes.NewReader(data), 1000, 999); err == nil || !strings.Contains(err.Error(), "ADR 0468 §3") { + t.Errorf("a size past the limit: %v", err) + } + if len(lengths) != 2 { + t.Errorf("a refused call made a request: the server saw %d", len(lengths)) + } + + // A body longer or shorter than declared fails, and the store is not told it succeeded. + reads := &http.Client{Transport: roundTrip(func(r *http.Request) (*http.Response, error) { + _, err := io.Copy(io.Discard, r.Body) + if err == nil { + return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(""))}, nil + } + return nil, err + })} + for name, declared := range map[string]int64{"longer than declared": 999, "shorter than declared": 1001} { + if _, err := Put(context.Background(), reads, store.URL+"/b/k", bytes.NewReader(data), declared, 0); err == nil { + t.Errorf("a body %s was accepted", name) + } + } + + _, err = Put(context.Background(), store.Client(), store.URL+"/b/k?deny=1", bytes.NewReader(data), 1000, 0) + e, ok := err.(*StoreError) + if !ok || e.Status != 403 || e.Code != "SignatureDoesNotMatch" || !strings.Contains(e.Error(), "SignatureDoesNotMatch") { + t.Fatalf("the refusal is %v", err) + } +} + +// 1592/AC4: Get writes the body and answers size and SHA-256; a wrong digest and a document past the limit are errors. +func TestGetStreamsChecksAndRefuses(t *testing.T) { + data := bytes.Repeat([]byte("abcdefghij"), 100) + store := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write(data) + })) + defer store.Close() + var out bytes.Buffer + got, err := Get(context.Background(), store.Client(), store.URL+"/b/k", &out, 0, hexSum(data)) + if err != nil { + t.Fatal(err) + } + if got.Size != 1000 || got.SHA256 != hexSum(data) || !bytes.Equal(out.Bytes(), data) { + t.Errorf("answered %+v, wrote %d bytes", got, out.Len()) + } + if _, err := Get(context.Background(), store.Client(), store.URL+"/b/k", io.Discard, 0, hexSum([]byte("other"))); err == nil { + t.Error("a wrong digest was accepted") + } + if _, err := Get(context.Background(), store.Client(), store.URL+"/b/k", io.Discard, 999, ""); err == nil { + t.Error("a document past the limit was accepted") + } + chunked := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.(http.Flusher).Flush() + w.Write(data) + })) + defer chunked.Close() + if _, err := Get(context.Background(), chunked.Client(), chunked.URL+"/b/k", io.Discard, 999, ""); err == nil { + t.Error("a chunked document past the limit was accepted") + } + if _, err := Get(context.Background(), chunked.Client(), chunked.URL+"/b/k", io.Discard, 1000, ""); err != nil { + t.Errorf("exactly the limit: %v", err) + } +}