mesh-catalog/modules/distribution/internal/holder/oci.go

package holder

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"sort"
	"strings"
)

// OCI is the artifact store as the OCI distribution protocol answers it on its node's loopback, written by
// distribution's own process alone: the Store and the Catalog the verbs use. It answers a reference's path,
// `<name>@sha256:<digest>`; the holder puts the seat's name before it (SeatReference, ADR 0605 §9).
type OCI struct {
	Base   *url.URL
	Client *http.Client
	// Upstream reads the outside registries `mirror` copies from.
	Upstream *Upstream
}

const (
	mediaManifest    = "application/vnd.oci.image.manifest.v1+json"
	mediaEmptyConfig = "application/vnd.oci.empty.v1+json"
	mediaLayer       = "application/vnd.oci.image.layer.v1.tar"
	emptyConfig      = "{}"
)

func (o *OCI) client() *http.Client {
	if o.Client != nil {
		return o.Client
	}
	return http.DefaultClient
}

func (o *OCI) url(path string) string { return strings.TrimSuffix(o.Base.String(), "/") + path }

func (o *OCI) do(method, path, contentType string, body []byte, accept ...string) (*http.Response, []byte, error) {
	req, err := http.NewRequest(method, o.url(path), bytes.NewReader(body))
	if err != nil {
		return nil, nil, err
	}
	if contentType != "" {
		req.Header.Set("Content-Type", contentType)
	}
	for _, a := range accept {
		req.Header.Add("Accept", a)
	}
	resp, err := o.client().Do(req)
	if err != nil {
		return nil, nil, err
	}
	defer resp.Body.Close()
	b, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<20))
	return resp, b, nil
}

// Has is whether name holds a blob.
func (o *OCI) Has(name, digest string) (bool, error) {
	resp, _, err := o.do(http.MethodHead, "/v2/"+name+"/blobs/"+digest, "", nil)
	if err != nil {
		return false, err
	}
	return resp.StatusCode == http.StatusOK, nil
}

func (o *OCI) putBlob(name, digest string, data []byte) error {
	resp, b, err := o.do(http.MethodPost, "/v2/"+name+"/blobs/uploads/", "", nil)
	if err != nil {
		return err
	}
	if resp.StatusCode != http.StatusAccepted {
		return fmt.Errorf("the store did not open an upload of %s (%d): %s", digest, resp.StatusCode, b)
	}
	loc, err := url.Parse(resp.Header.Get("Location"))
	if err != nil {
		return err
	}
	q := loc.Query()
	q.Set("digest", digest)
	loc.RawQuery = q.Encode()
	path := loc.Path
	if loc.RawQuery != "" {
		path += "?" + loc.RawQuery
	}
	resp, b, err = o.do(http.MethodPut, path, "application/octet-stream", data)
	if err != nil {
		return err
	}
	if resp.StatusCode != http.StatusCreated {
		return fmt.Errorf("the store did not take %s (%d): %s", digest, resp.StatusCode, b)
	}
	return nil
}

func (o *OCI) putManifest(name, ref, mediaType string, data []byte) (string, error) {
	resp, b, err := o.do(http.MethodPut, "/v2/"+name+"/manifests/"+ref, mediaType, data)
	if err != nil {
		return "", err
	}
	if resp.StatusCode != http.StatusCreated {
		return "", fmt.Errorf("the store did not take the manifest %s:%s (%d): %s", name, ref, resp.StatusCode, b)
	}
	return "sha256:" + sum(data), nil
}

// Put stores a transfer: an image's layers and config as blobs and its manifest (the part whose media type is a
// manifest's) under its version; a bundle's or an archive's one blob under an artifact manifest of an empty config and
// one layer (the shape the artifact sweep recognises), under its version, or by its digest alone when unnumbered.
func (o *OCI) Put(t PublishRequest, parts map[string][]byte) (string, error) {
	var manifest *Part
	digests := make([]string, 0, len(parts))
	for d := range parts {
		digests = append(digests, d)
	}
	sort.Strings(digests)
	for i, p := range t.Parts {
		// An image's manifest is the part whose digest is the publish's (the seat's contract names no media type); a
		// part naming a manifest's media type is one too.
		if (t.Kind == KindImage || t.Kind == KindUpstream) && (p.Digest == t.Digest || strings.Contains(p.MediaType, "manifest")) {
			manifest = &t.Parts[i]
			continue
		}
		if data, ok := parts[p.Digest]; ok {
			if err := o.putBlob(t.Name, p.Digest, data); err != nil {
				return "", err
			}
		}
	}
	ref := t.Version
	switch t.Kind {
	case KindImage, KindUpstream:
		if manifest == nil {
			return "", fmt.Errorf("an image's publish names no manifest part")
		}
		data, ok := parts[manifest.Digest]
		if !ok {
			// Held already: fetch it to tag it under the version.
			resp, b, err := o.do(http.MethodGet, "/v2/"+t.Name+"/manifests/"+manifest.Digest, "", nil, accepted...)
			if err != nil || resp.StatusCode != http.StatusOK {
				return "", fmt.Errorf("the held manifest %s cannot be read: %v", manifest.Digest, err)
			}
			data = b
		}
		if ref == "" {
			ref = manifest.Digest
		}
		mt := manifest.MediaType
		var probe struct {
			MediaType string `json:"mediaType"`
		}
		if mt == "" && json.Unmarshal(data, &probe) == nil {
			mt = probe.MediaType
		}
		if mt == "" {
			mt = mediaManifest
		}
		d, err := o.putManifest(t.Name, ref, mt, data)
		if err != nil {
			return "", err
		}
		return t.Name + "@" + d, nil
	default:
		if err := o.putBlob(t.Name, "sha256:"+sum([]byte(emptyConfig)), []byte(emptyConfig)); err != nil {
			return "", err
		}
		size, err := o.layerSize(t, parts)
		if err != nil {
			return "", err
		}
		m, _ := json.Marshal(map[string]any{
			"schemaVersion": 2, "mediaType": mediaManifest,
			"config": map[string]any{"mediaType": mediaEmptyConfig, "digest": "sha256:" + sum([]byte(emptyConfig)), "size": len(emptyConfig)},
			"layers": []map[string]any{{"mediaType": mediaLayer, "digest": t.Digest, "size": size}},
		})
		if ref == "" {
			ref = "sha256:" + sum(m)
		}
		if _, err := o.putManifest(t.Name, ref, mediaManifest, m); err != nil {
			return "", err
		}
		return t.Name + "@" + t.Digest, nil
	}
}

// layerSize is the size of a bundle's or an archive's one blob: its bytes' when they travelled, else the size its part
// names, else (a late publish of a held digest names it with no size, its bytes having left the build node) the store's.
func (o *OCI) layerSize(t PublishRequest, parts map[string][]byte) (int64, error) {
	if data, ok := parts[t.Digest]; ok {
		return int64(len(data)), nil
	}
	for _, p := range t.Parts {
		if p.Digest == t.Digest && p.Size > 0 {
			return p.Size, nil
		}
	}
	resp, _, err := o.do(http.MethodHead, "/v2/"+t.Name+"/blobs/"+t.Digest, "", nil)
	if err != nil {
		return 0, err
	}
	if resp.StatusCode != http.StatusOK || resp.ContentLength < 0 {
		return 0, fmt.Errorf("the store holds no blob %s of %s to tag (%d)", t.Digest, t.Name, resp.StatusCode)
	}
	return resp.ContentLength, nil
}

// Held is the reference of name at version when the store holds digest under it.
func (o *OCI) Held(name, version, digest string) (string, bool, error) {
	d, err := o.Digest(name, version)
	if err != nil || d == "" {
		return "", false, err
	}
	if d != digest {
		return "", false, nil
	}
	return name + "@" + d, true, nil
}

// Versions are name's tags.
func (o *OCI) Versions(name string) ([]string, error) {
	resp, b, err := o.do(http.MethodGet, "/v2/"+name+"/tags/list", "", nil)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode == http.StatusNotFound {
		return []string{}, nil
	}
	var out struct {
		Tags []string `json:"tags"`
	}
	if resp.StatusCode != http.StatusOK || json.Unmarshal(b, &out) != nil {
		return nil, fmt.Errorf("the store's tags of %s (%d): %s", name, resp.StatusCode, b)
	}
	sort.Strings(out.Tags)
	return out.Tags, nil
}

// Digest is the manifest digest name holds at version, empty when none.
func (o *OCI) Digest(name, version string) (string, error) {
	resp, _, err := o.do(http.MethodHead, "/v2/"+name+"/manifests/"+version, "", nil, mediaManifest,
		"application/vnd.oci.image.index.v1+json", "application/vnd.docker.distribution.manifest.v2+json")
	if err != nil {
		return "", err
	}
	if resp.StatusCode == http.StatusNotFound {
		return "", nil
	}
	if resp.StatusCode != http.StatusOK {
		return "", fmt.Errorf("the store answered %d for %s:%s", resp.StatusCode, name, version)
	}
	return resp.Header.Get("Docker-Content-Digest"), nil
}

// ErrNoTagAlone is a store that cannot delete a tag alone. registry 2.8.3, the store's pinned image, is one: its
// DeleteManifest (distribution v2.8.3 registry/handlers/manifests.go) ignores a tag reference and deletes the empty
// digest, whose check fails, answered 400 DIGEST_INVALID; registry 3 untags alone.
var ErrNoTagAlone = errors.New("the store cannot remove a tag alone")

// RemoveTag deletes the tag version of name alone (the OCI distribution protocol's DELETE of a manifest by its tag);
// the manifest stays under every other tag. A store that cannot delete a tag answers ErrNoTagAlone, any other failure
// an error of its own; either removes nothing.
func (o *OCI) RemoveTag(name, version string) error {
	resp, b, err := o.do(http.MethodDelete, "/v2/"+name+"/manifests/"+version, "", nil)
	if err != nil {
		return err
	}
	if resp.StatusCode == http.StatusBadRequest && bytes.Contains(b, []byte(`"DIGEST_INVALID"`)) {
		return fmt.Errorf("%w: DELETE of %s:%s by its tag answered 400 DIGEST_INVALID", ErrNoTagAlone, name, version)
	}
	if resp.StatusCode != http.StatusAccepted && resp.StatusCode != http.StatusNotFound {
		return fmt.Errorf("the store did not delete the tag %s:%s alone (%d): %s", name, version, resp.StatusCode, b)
	}
	return nil
}

// Remove deletes the manifest name holds at version, by its digest: every tag naming it goes with it.
func (o *OCI) Remove(name, version string) error {
	d, err := o.Digest(name, version)
	if err != nil || d == "" {
		return err
	}
	return o.RemoveDigest(name, d)
}

// RemoveDigest deletes a manifest by its digest; the store's nightly collector frees its blobs.
func (o *OCI) RemoveDigest(name, digest string) error {
	resp, b, err := o.do(http.MethodDelete, "/v2/"+name+"/manifests/"+digest, "", nil)
	if err != nil {
		return err
	}
	if resp.StatusCode != http.StatusAccepted && resp.StatusCode != http.StatusNotFound {
		return fmt.Errorf("the store did not delete %s@%s (%d): %s", name, digest, resp.StatusCode, b)
	}
	return nil
}