Files
mesh-catalog/modules/distribution/internal/holder/oci.go
T
jschoubben 4a99f7044e distribution: a publish with a version and no parts tags the digest the store holds
The builder's publish again of mesh-delivery-pipeline 0.1.6 sends the version and the digest with no parts, as A-73
has it; the build stored the bundle untagged, and the holder answered 'publish: no parts' unless the tag existed
already. The holder now checks the digest against the store under the name (the artifact store, A-50), asks the
builder's reservation, tags it with the version and says published; a digest it holds none of is refused as not
stored, naming the digest.

Builds: design 47 G A-417, A-73, A-50, §588
2026-11-15 02:39:28 +01:00

318 lines
10 KiB
Go

package holder
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"sort"
"strings"
)
// OCI is the artifact store as the OCI distribution protocol answers it on its node's loopback, written by
// distribution's own process alone: the Store and the Catalog the verbs use. It answers a reference's path,
// `<name>@sha256:<digest>`; the holder puts the seat's name before it (SeatReference, ADR 0605 §9).
type OCI struct {
Base *url.URL
Client *http.Client
// Upstream reads the outside registries `mirror` copies from.
Upstream *Upstream
}
const (
mediaManifest = "application/vnd.oci.image.manifest.v1+json"
mediaEmptyConfig = "application/vnd.oci.empty.v1+json"
mediaLayer = "application/vnd.oci.image.layer.v1.tar"
emptyConfig = "{}"
)
func (o *OCI) client() *http.Client {
if o.Client != nil {
return o.Client
}
return http.DefaultClient
}
func (o *OCI) url(path string) string { return strings.TrimSuffix(o.Base.String(), "/") + path }
func (o *OCI) do(method, path, contentType string, body []byte, accept ...string) (*http.Response, []byte, error) {
req, err := http.NewRequest(method, o.url(path), bytes.NewReader(body))
if err != nil {
return nil, nil, err
}
if contentType != "" {
req.Header.Set("Content-Type", contentType)
}
for _, a := range accept {
req.Header.Add("Accept", a)
}
resp, err := o.client().Do(req)
if err != nil {
return nil, nil, err
}
defer resp.Body.Close()
b, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<20))
return resp, b, nil
}
// Has is whether name holds a blob.
func (o *OCI) Has(name, digest string) (bool, error) {
resp, _, err := o.do(http.MethodHead, "/v2/"+name+"/blobs/"+digest, "", nil)
if err != nil {
return false, err
}
return resp.StatusCode == http.StatusOK, nil
}
func (o *OCI) putBlob(name, digest string, data []byte) error {
resp, b, err := o.do(http.MethodPost, "/v2/"+name+"/blobs/uploads/", "", nil)
if err != nil {
return err
}
if resp.StatusCode != http.StatusAccepted {
return fmt.Errorf("the store did not open an upload of %s (%d): %s", digest, resp.StatusCode, b)
}
loc, err := url.Parse(resp.Header.Get("Location"))
if err != nil {
return err
}
q := loc.Query()
q.Set("digest", digest)
loc.RawQuery = q.Encode()
path := loc.Path
if loc.RawQuery != "" {
path += "?" + loc.RawQuery
}
resp, b, err = o.do(http.MethodPut, path, "application/octet-stream", data)
if err != nil {
return err
}
if resp.StatusCode != http.StatusCreated {
return fmt.Errorf("the store did not take %s (%d): %s", digest, resp.StatusCode, b)
}
return nil
}
func (o *OCI) putManifest(name, ref, mediaType string, data []byte) (string, error) {
resp, b, err := o.do(http.MethodPut, "/v2/"+name+"/manifests/"+ref, mediaType, data)
if err != nil {
return "", err
}
if resp.StatusCode != http.StatusCreated {
return "", fmt.Errorf("the store did not take the manifest %s:%s (%d): %s", name, ref, resp.StatusCode, b)
}
return "sha256:" + sum(data), nil
}
// Put stores a transfer: an image's layers and config as blobs and its manifest (the part whose media type is a
// manifest's) under its version; a bundle's or an archive's one blob under an artifact manifest of an empty config and
// one layer (the shape the artifact sweep recognises), under its version, or by its digest alone when unnumbered.
func (o *OCI) Put(t PublishRequest, parts map[string][]byte) (string, error) {
var manifest *Part
digests := make([]string, 0, len(parts))
for d := range parts {
digests = append(digests, d)
}
sort.Strings(digests)
for i, p := range t.Parts {
// An image's manifest is the part whose digest is the publish's (the seat's contract names no media type); a
// part naming a manifest's media type is one too.
if (t.Kind == KindImage || t.Kind == KindUpstream) && (p.Digest == t.Digest || strings.Contains(p.MediaType, "manifest")) {
manifest = &t.Parts[i]
continue
}
if data, ok := parts[p.Digest]; ok {
if err := o.putBlob(t.Name, p.Digest, data); err != nil {
return "", err
}
}
}
ref := t.Version
switch t.Kind {
case KindImage, KindUpstream:
if manifest == nil {
return "", fmt.Errorf("an image's publish names no manifest part")
}
data, ok := parts[manifest.Digest]
if !ok {
// Held already: fetch it to tag it under the version.
resp, b, err := o.do(http.MethodGet, "/v2/"+t.Name+"/manifests/"+manifest.Digest, "", nil, accepted...)
if err != nil || resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("the held manifest %s cannot be read: %v", manifest.Digest, err)
}
data = b
}
if ref == "" {
ref = manifest.Digest
}
mt := manifest.MediaType
var probe struct {
MediaType string `json:"mediaType"`
}
if mt == "" && json.Unmarshal(data, &probe) == nil {
mt = probe.MediaType
}
if mt == "" {
mt = mediaManifest
}
d, err := o.putManifest(t.Name, ref, mt, data)
if err != nil {
return "", err
}
return t.Name + "@" + d, nil
default:
if err := o.putBlob(t.Name, "sha256:"+sum([]byte(emptyConfig)), []byte(emptyConfig)); err != nil {
return "", err
}
size, err := o.layerSize(t, parts)
if err != nil {
return "", err
}
m, _ := json.Marshal(map[string]any{
"schemaVersion": 2, "mediaType": mediaManifest,
"config": map[string]any{"mediaType": mediaEmptyConfig, "digest": "sha256:" + sum([]byte(emptyConfig)), "size": len(emptyConfig)},
"layers": []map[string]any{{"mediaType": mediaLayer, "digest": t.Digest, "size": size}},
})
if ref == "" {
ref = "sha256:" + sum(m)
}
if _, err := o.putManifest(t.Name, ref, mediaManifest, m); err != nil {
return "", err
}
return t.Name + "@" + t.Digest, nil
}
}
// layerSize is the size of a bundle's or an archive's one blob: its bytes' when they travelled, else the size its part
// names, else (a late publish of a held digest names it with no size, its bytes having left the build node) the store's.
func (o *OCI) layerSize(t PublishRequest, parts map[string][]byte) (int64, error) {
if data, ok := parts[t.Digest]; ok {
return int64(len(data)), nil
}
for _, p := range t.Parts {
if p.Digest == t.Digest && p.Size > 0 {
return p.Size, nil
}
}
resp, _, err := o.do(http.MethodHead, "/v2/"+t.Name+"/blobs/"+t.Digest, "", nil)
if err != nil {
return 0, err
}
if resp.StatusCode != http.StatusOK || resp.ContentLength < 0 {
return 0, fmt.Errorf("the store holds no blob %s of %s to tag (%d)", t.Digest, t.Name, resp.StatusCode)
}
return resp.ContentLength, nil
}
// Held is the reference of name at version when the store holds digest under it.
func (o *OCI) Held(name, version, digest string) (string, bool, error) {
d, err := o.Digest(name, version)
if err != nil || d == "" {
return "", false, err
}
if d != digest {
return "", false, nil
}
return name + "@" + d, true, nil
}
// Holds is whether the store holds what r's digest names under r's name: an image's manifest by its digest, or a bundle's
// or an archive's blob.
func (o *OCI) Holds(r PublishRequest) (bool, error) {
if r.Kind != KindImage && r.Kind != KindUpstream {
return o.Has(r.Name, r.Digest)
}
resp, _, err := o.do(http.MethodHead, "/v2/"+r.Name+"/manifests/"+r.Digest, "", nil, accepted...)
if err != nil {
return false, err
}
switch resp.StatusCode {
case http.StatusOK:
return true, nil
case http.StatusNotFound:
return false, nil
}
return false, fmt.Errorf("the store answered %d for the manifest %s of %s", resp.StatusCode, r.Digest, r.Name)
}
// Versions are name's tags.
func (o *OCI) Versions(name string) ([]string, error) {
resp, b, err := o.do(http.MethodGet, "/v2/"+name+"/tags/list", "", nil)
if err != nil {
return nil, err
}
if resp.StatusCode == http.StatusNotFound {
return []string{}, nil
}
var out struct {
Tags []string `json:"tags"`
}
if resp.StatusCode != http.StatusOK || json.Unmarshal(b, &out) != nil {
return nil, fmt.Errorf("the store's tags of %s (%d): %s", name, resp.StatusCode, b)
}
sort.Strings(out.Tags)
return out.Tags, nil
}
// Digest is the manifest digest name holds at version, empty when none.
func (o *OCI) Digest(name, version string) (string, error) {
resp, _, err := o.do(http.MethodHead, "/v2/"+name+"/manifests/"+version, "", nil, mediaManifest,
"application/vnd.oci.image.index.v1+json", "application/vnd.docker.distribution.manifest.v2+json")
if err != nil {
return "", err
}
if resp.StatusCode == http.StatusNotFound {
return "", nil
}
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("the store answered %d for %s:%s", resp.StatusCode, name, version)
}
return resp.Header.Get("Docker-Content-Digest"), nil
}
// ErrNoTagAlone is a store that cannot delete a tag alone. registry 2.8.3, the store's pinned image, is one: its
// DeleteManifest (distribution v2.8.3 registry/handlers/manifests.go) ignores a tag reference and deletes the empty
// digest, whose check fails, answered 400 DIGEST_INVALID; registry 3 untags alone.
var ErrNoTagAlone = errors.New("the store cannot remove a tag alone")
// RemoveTag deletes the tag version of name alone (the OCI distribution protocol's DELETE of a manifest by its tag);
// the manifest stays under every other tag. A store that cannot delete a tag answers ErrNoTagAlone, any other failure
// an error of its own; either removes nothing.
func (o *OCI) RemoveTag(name, version string) error {
resp, b, err := o.do(http.MethodDelete, "/v2/"+name+"/manifests/"+version, "", nil)
if err != nil {
return err
}
if resp.StatusCode == http.StatusBadRequest && bytes.Contains(b, []byte(`"DIGEST_INVALID"`)) {
return fmt.Errorf("%w: DELETE of %s:%s by its tag answered 400 DIGEST_INVALID", ErrNoTagAlone, name, version)
}
if resp.StatusCode != http.StatusAccepted && resp.StatusCode != http.StatusNotFound {
return fmt.Errorf("the store did not delete the tag %s:%s alone (%d): %s", name, version, resp.StatusCode, b)
}
return nil
}
// Remove deletes the manifest name holds at version, by its digest: every tag naming it goes with it.
func (o *OCI) Remove(name, version string) error {
d, err := o.Digest(name, version)
if err != nil || d == "" {
return err
}
return o.RemoveDigest(name, d)
}
// RemoveDigest deletes a manifest by its digest; the store's nightly collector frees its blobs.
func (o *OCI) RemoveDigest(name, digest string) error {
resp, b, err := o.do(http.MethodDelete, "/v2/"+name+"/manifests/"+digest, "", nil)
if err != nil {
return err
}
if resp.StatusCode != http.StatusAccepted && resp.StatusCode != http.StatusNotFound {
return fmt.Errorf("the store did not delete %s@%s (%d): %s", name, digest, resp.StatusCode, b)
}
return nil
}