package main // The module's bucket (novox/hq ADR 0468 clause 2): each scan's PDF kept once, by its SHA-256, and each upload // under uploads/, in the bucket the `minio` module provides this module through the `s3-bucket` provision. The // mesh writes the bucket's facts and its key into one file, store.json, which only the device process (account // brother-device) can read (BROTHER_STORE); this speaks S3 to it with the standard library alone, as mesh-mail's // store does (modules/mesh-mail/cmd/mesh-mail/objects.go): a HEAD, a PUT and a GET, each signed with AWS // Signature Version 4, path-style. import ( "bytes" "context" "crypto/hmac" "crypto/sha256" "encoding/hex" "encoding/json" "encoding/xml" "errors" "fmt" "io" "net/http" "net/url" "os" "sort" "strings" "time" "git.novox.be/novox/mesh-sdk/go/objectlink" ) // Objects is the module's bucket. type Objects interface { // Bucket is the bucket's name, as an event names it. Bucket() string // Has says whether the key is kept. Has(ctx context.Context, key string) (bool, error) // Put keeps the bytes under the key. Put(ctx context.Context, key, mediaType string, data []byte) error // Get is the bytes kept under the key. Get(ctx context.Context, key string) ([]byte, error) // List is the keys kept that start with the prefix, in order (the first thousand). List(ctx context.Context, prefix string) ([]string, error) // Delete removes the key; a key not kept is not an error. Delete(ctx context.Context, key string) error } // Store keeps the bytes under the key once: a key already kept is left as it is, since a key that names the // content (the SHA-256 of a scan) names the same bytes. func Store(ctx context.Context, o Objects, key, mediaType string, data []byte) error { has, err := o.Has(ctx, key) if err != nil { return err } if has { return nil } return o.Put(ctx, key, mediaType, data) } // Presigner is a store that signs a link to one of its objects (novox/hq ADR 0468 ยง3): the device process's, with the // binding's key, which reaches this module's bucket only. type Presigner interface { // Link is a link for GET or PUT of one key that lives for the given time (at most ten minutes). Link(method, key string, expires time.Duration) (string, error) } // StoreFacts is the file the mesh writes from the s3-bucket binding and its secret. type StoreFacts struct { Scheme string `json:"scheme"` At string `json:"at"` Port string `json:"port"` Bucket string `json:"bucket"` Region string `json:"region"` Key string `json:"key"` Secret string `json:"secret"` } // ReadStoreFacts reads the store's facts. func ReadStoreFacts(path string) (StoreFacts, error) { var f StoreFacts if path == "" { return f, errors.New("the module was started without its store's facts (BROTHER_STORE)") } raw, err := os.ReadFile(path) if err != nil { return f, fmt.Errorf("the store's facts cannot be read: %w", err) } if err := json.Unmarshal(raw, &f); err != nil { return f, fmt.Errorf("the store's facts are not the module's: %v", err) } if f.At == "" || f.Bucket == "" || f.Key == "" || f.Secret == "" { return f, errors.New("the store's facts name no address, bucket or key") } if f.Scheme == "" { f.Scheme = "http" } if f.Region == "" { f.Region = "us-east-1" } return f, nil } // S3 is the module's bucket over S3. type S3 struct { Endpoint string // scheme://host[:port] Name string Region string Access string Secret string Client *http.Client Now func() time.Time } // NewS3 is the store the facts name. func NewS3(f StoreFacts) *S3 { host := f.At if f.Port != "" { host += ":" + f.Port } return &S3{Endpoint: f.Scheme + "://" + host, Name: f.Bucket, Region: f.Region, Access: f.Key, Secret: f.Secret, Client: &http.Client{Timeout: 2 * time.Minute}, Now: time.Now} } func (s *S3) Bucket() string { return s.Name } // Link signs a link for one object of the bucket with the binding's key, for the host the binding gives. func (s *S3) Link(method, key string, expires time.Duration) (string, error) { now := time.Now if s.Now != nil { now = s.Now } return objectlink.Presign(objectlink.Request{Method: method, Endpoint: s.Endpoint, Bucket: s.Name, Key: key, AccessKey: s.Access, SecretKey: s.Secret, Region: s.Region, Expires: expires, Now: now()}) } func (s *S3) do(ctx context.Context, method, key, mediaType string, body []byte) (*http.Response, error) { u, err := url.Parse(s.Endpoint + "/" + s.Name + "/" + key) if err != nil { return nil, err } req, err := http.NewRequestWithContext(ctx, method, u.String(), bytes.NewReader(body)) if err != nil { return nil, err } if mediaType != "" { req.Header.Set("Content-Type", mediaType) } req.ContentLength = int64(len(body)) Sign(req, body, s.Access, s.Secret, s.Region, s.Now().UTC()) return s.Client.Do(req) } func (s *S3) Has(ctx context.Context, key string) (bool, error) { resp, err := s.do(ctx, http.MethodHead, key, "", nil) if err != nil { return false, fmt.Errorf("the module's bucket cannot be reached: %w", err) } resp.Body.Close() switch resp.StatusCode { case http.StatusOK: return true, nil case http.StatusNotFound: return false, nil } return false, fmt.Errorf("the module's bucket answered %s to asking for %s", resp.Status, key) } func (s *S3) Put(ctx context.Context, key, mediaType string, data []byte) error { resp, err := s.do(ctx, http.MethodPut, key, mediaType, data) if err != nil { return fmt.Errorf("the module's bucket cannot be reached: %w", err) } defer resp.Body.Close() if resp.StatusCode/100 != 2 { said, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) return fmt.Errorf("the module's bucket refused %s: %s %s", key, resp.Status, strings.TrimSpace(string(said))) } return nil } func (s *S3) Get(ctx context.Context, key string) ([]byte, error) { resp, err := s.do(ctx, http.MethodGet, key, "", nil) if err != nil { return nil, fmt.Errorf("the module's bucket cannot be reached: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("the module's bucket answered %s for %s", resp.Status, key) } return io.ReadAll(resp.Body) } func (s *S3) List(ctx context.Context, prefix string) ([]string, error) { resp, err := s.do(ctx, http.MethodGet, "?list-type=2&prefix="+url.QueryEscape(prefix), "", nil) if err != nil { return nil, fmt.Errorf("the module's bucket cannot be reached: %w", err) } defer resp.Body.Close() body, _ := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("the module's bucket answered %s to listing %s", resp.Status, prefix) } var out struct { Keys []string `xml:"Contents>Key"` } if err := xml.Unmarshal(body, &out); err != nil { return nil, fmt.Errorf("the module's bucket listed %s in a form that cannot be read: %w", prefix, err) } return out.Keys, nil } func (s *S3) Delete(ctx context.Context, key string) error { resp, err := s.do(ctx, http.MethodDelete, key, "", nil) if err != nil { return fmt.Errorf("the module's bucket cannot be reached: %w", err) } defer resp.Body.Close() if resp.StatusCode/100 != 2 && resp.StatusCode != http.StatusNotFound { return fmt.Errorf("the module's bucket refused to delete %s: %s", key, resp.Status) } return nil } // Sign signs a request with AWS Signature Version 4 for S3: every header it carries but the agent and the length is // signed, with the payload's hash. func Sign(req *http.Request, body []byte, access, secret, region string, now time.Time) { payload := sha256.Sum256(body) payloadHex := hex.EncodeToString(payload[:]) amzDate := now.Format("20060102T150405Z") day := now.Format("20060102") req.Header.Set("x-amz-date", amzDate) req.Header.Set("x-amz-content-sha256", payloadHex) headers := map[string]string{"host": req.URL.Host} for k, v := range req.Header { lk := strings.ToLower(k) if lk == "user-agent" || lk == "content-length" || lk == "authorization" { continue } headers[lk] = strings.TrimSpace(strings.Join(v, ",")) } names := make([]string, 0, len(headers)) for k := range headers { names = append(names, k) } sort.Strings(names) var canonHeaders strings.Builder for _, k := range names { canonHeaders.WriteString(k + ":" + headers[k] + "\n") } signed := strings.Join(names, ";") path := req.URL.EscapedPath() if path == "" { path = "/" } canonical := strings.Join([]string{req.Method, path, canonicalQuery(req.URL.Query()), canonHeaders.String(), signed, payloadHex}, "\n") scope := day + "/" + region + "/s3/aws4_request" sum := sha256.Sum256([]byte(canonical)) toSign := "AWS4-HMAC-SHA256\n" + amzDate + "\n" + scope + "\n" + hex.EncodeToString(sum[:]) key := hmacOf([]byte("AWS4"+secret), day) key = hmacOf(key, region) key = hmacOf(key, "s3") key = hmacOf(key, "aws4_request") sig := hex.EncodeToString(hmacOf(key, toSign)) req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential="+access+"/"+scope+", SignedHeaders="+signed+", Signature="+sig) } func canonicalQuery(q url.Values) string { keys := make([]string, 0, len(q)) for k := range q { keys = append(keys, k) } sort.Strings(keys) var parts []string for _, k := range keys { vs := append([]string(nil), q[k]...) sort.Strings(vs) for _, v := range vs { parts = append(parts, url.QueryEscape(k)+"="+strings.ReplaceAll(url.QueryEscape(v), "+", "%20")) } } return strings.Join(parts, "&") } func hmacOf(key []byte, data string) []byte { h := hmac.New(sha256.New, key) h.Write([]byte(data)) return h.Sum(nil) }