mesh-catalog/modules/distribution/internal/holder/files.go

package holder

// **The store's own files, read through its own container** (novox/hq ADR 0632 §3): what `list` answers. The store's
// read side lists repositories and the tags in each, but not a manifest no tag names, and since the mesh pins every
// machine by digest that is almost every manifest the store holds. The files list them all, with every blob's size, and
// hold every manifest's content. They are read through the store's container, distribution's own (the resource store),
// and only read; the artifact sweep and the read tools ask `list` and never reach the files themselves. Moved here from
// artifact-store-tools, which read them by docker exec into this module's container until ADR 0632.

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"os/exec"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
)

// storageRoot is where the registry keeps its files, inside its container.
const storageRoot = "/var/lib/registry/docker/registry/v2"

// listScript lists, in three sections, every blob with its size, every manifest each repository holds,
// and what each tag names now. Busybox's find and stat (the store's image is Alpine): no -printf. An
// upload in progress is not listed, and nor are a tag's former targets.
const listScript = `set -e
cd ` + storageRoot + `
echo '#blobs'
if [ -d blobs ]; then find blobs -type f -name data | xargs -r stat -c '%s %n'; fi
echo '#revisions'
if [ -d repositories ]; then find repositories -type f -path '*/_manifests/revisions/sha256/*/link'; fi
echo '#tags'
if [ -d repositories ]; then find repositories -type f -path '*/_manifests/tags/*/current/link' -exec grep -H . {} + || true; fi
echo '#end'`

// Repo is one repository as the store's files list it.
type Repo struct {
	Name string
	// Revisions are the digests of every manifest it holds, tagged or not.
	Revisions map[string]bool
	// Tags are what each tag names now.
	Tags map[string]string
}

// Listing is the store's files: every blob with its size, and every repository.
type Listing struct {
	Blobs map[string]int64
	Repos map[string]*Repo
}

func (l *Listing) repo(name string) *Repo {
	r := l.Repos[name]
	if r == nil {
		r = &Repo{Name: name, Revisions: map[string]bool{}, Tags: map[string]string{}}
		l.Repos[name] = r
	}
	return r
}

// RepoNames are the repositories in order.
func (l *Listing) RepoNames() []string {
	out := make([]string, 0, len(l.Repos))
	for n := range l.Repos {
		out = append(out, n)
	}
	sort.Strings(out)
	return out
}

// ParseListing reads listScript's output. A section that never ended means the listing was cut short,
// which is an error: a partial listing would make held things look absent.
func ParseListing(out string) (*Listing, error) {
	l := &Listing{Blobs: map[string]int64{}, Repos: map[string]*Repo{}}
	section := ""
	ended := false
	for _, line := range strings.Split(out, "\n") {
		line = strings.TrimSpace(line)
		if line == "" {
			continue
		}
		if strings.HasPrefix(line, "#") {
			section = line
			ended = line == "#end"
			continue
		}
		switch section {
		case "#blobs":
			size, path, ok := strings.Cut(line, " ")
			n, err := strconv.ParseInt(size, 10, 64)
			if !ok || err != nil {
				return nil, fmt.Errorf("the store's listing has a blob line it cannot read: %q", line)
			}
			// blobs/sha256/ab/<hex>/data
			parts := strings.Split(strings.TrimPrefix(path, "./"), "/")
			if len(parts) != 5 || parts[0] != "blobs" || parts[4] != "data" {
				return nil, fmt.Errorf("the store's listing has a blob at an unexpected place: %q", path)
			}
			l.Blobs[parts[1]+":"+parts[3]] = n
		case "#revisions":
			// repositories/<repo>/_manifests/revisions/sha256/<hex>/link
			name, rest, ok := strings.Cut(strings.TrimPrefix(strings.TrimPrefix(line, "./"), "repositories/"), "/_manifests/revisions/")
			parts := strings.Split(rest, "/")
			if !ok || len(parts) != 3 || parts[2] != "link" {
				return nil, fmt.Errorf("the store's listing has a manifest at an unexpected place: %q", line)
			}
			l.repo(name).Revisions[parts[0]+":"+parts[1]] = true
		case "#tags":
			// repositories/<repo>/_manifests/tags/<tag>/current/link:sha256:<hex>
			path, digest, ok := strings.Cut(line, ":sha256:")
			name, rest, ok2 := strings.Cut(strings.TrimPrefix(strings.TrimPrefix(path, "./"), "repositories/"), "/_manifests/tags/")
			tag, _, ok3 := strings.Cut(rest, "/current/link")
			if !ok || !ok2 || !ok3 || tag == "" {
				return nil, fmt.Errorf("the store's listing has a tag line it cannot read: %q", line)
			}
			l.repo(name).Tags[tag] = "sha256:" + digest
		default:
			return nil, fmt.Errorf("the store's listing has a line outside any section: %q", line)
		}
	}
	if !ended {
		return nil, fmt.Errorf("the store's listing was cut short (it never reached its end)")
	}
	return l, nil
}

// Manifest is what one manifest names.
type Manifest struct {
	MediaType   string
	ConfigMedia string
	// Refs are the blobs it names: its configuration and its layers.
	Refs []string
	// Layers are its layers alone, in order.
	Layers []string
	// Children are the manifests an index names.
	Children []string
}

// Holder is the shape of a manifest that keeps one blob: an empty configuration and one layer. The
// controller holds every archive the mesh keeps this way (novox/hq issue 253), and keeps a named
// document the same way, under a tag.
func (m *Manifest) Holder() bool {
	return m != nil && m.ConfigMedia == "application/vnd.oci.empty.v1+json" && len(m.Layers) == 1
}

// ParseManifest reads a manifest's content, of any of the shapes a registry holds.
func ParseManifest(raw []byte) (*Manifest, error) {
	type desc struct {
		MediaType string `json:"mediaType"`
		Digest    string `json:"digest"`
	}
	var v struct {
		MediaType string `json:"mediaType"`
		Config    *desc  `json:"config"`
		Layers    []desc `json:"layers"`
		Manifests []desc `json:"manifests"`
		FSLayers  []struct {
			BlobSum string `json:"blobSum"`
		} `json:"fsLayers"`
	}
	if err := json.Unmarshal(raw, &v); err != nil {
		return nil, fmt.Errorf("not a manifest: %v", err)
	}
	m := &Manifest{MediaType: v.MediaType}
	if v.Config != nil && v.Config.Digest != "" {
		m.ConfigMedia = v.Config.MediaType
		m.Refs = append(m.Refs, v.Config.Digest)
	}
	for _, d := range v.Layers {
		m.Refs = append(m.Refs, d.Digest)
		m.Layers = append(m.Layers, d.Digest)
	}
	for _, d := range v.FSLayers {
		m.Refs = append(m.Refs, d.BlobSum)
		m.Layers = append(m.Layers, d.BlobSum)
	}
	for _, d := range v.Manifests {
		m.Children = append(m.Children, d.Digest)
	}
	return m, nil
}

// readScript prints each manifest named on its command line, each after a line naming its path. A
// manifest is JSON, which holds no raw line break inside a string, so no line of one starts with the
// marker. One the store does not have is said, not skipped.
const readScript = `cd ` + storageRoot + `
for p in "$@"; do
  echo "#manifest $p"
  if [ -f "$p" ]; then cat "$p"; echo; else echo "#absent"; fi
done
echo '#end'`

// readBatch is how many manifests one exec reads: well under any command line's limit.
const readBatch = 400

// Files reaches the artifact store's own files, through its own container, and only reads them.
type Files struct {
	Container string
	Run       Runner
	UID       int

	mu sync.Mutex
	// cache holds every manifest read: a manifest is named by its content's digest, so it never changes.
	cache map[string]*Manifest
}

// List reads the store's files.
func (s *Files) List(ctx context.Context) (*Listing, error) {
	if s.Container == "" {
		return nil, fmt.Errorf("this bundle was not told the artifact store's container (MESH_REGISTRY_CONTAINER)")
	}
	out, err := docker(ctx, s.Run, s.UID, "exec", s.Container, "sh", "-c", listScript)
	if err != nil {
		return nil, err
	}
	return ParseListing(out)
}

// Key names one manifest in one repository.
func Key(repo, digest string) string { return repo + "@" + digest }

// blobPath is where the store keeps a blob, under storageRoot.
func blobPath(digest string) string {
	algorithm, hex, _ := strings.Cut(digest, ":")
	if algorithm == "" || len(hex) < 2 || strings.ContainsAny(digest, "/ \n") {
		return ""
	}
	return "blobs/" + algorithm + "/" + hex[:2] + "/" + hex + "/data"
}

// Manifests reads every manifest the listing names, from the cache or the store's files. Answers each
// one read, and each one not read with why.
func (s *Files) Manifests(ctx context.Context, l *Listing) (map[string]*Manifest, map[string]string) {
	need := map[string]bool{}
	s.mu.Lock()
	if s.cache == nil {
		s.cache = map[string]*Manifest{}
	}
	for _, name := range l.RepoNames() {
		for d := range l.Repos[name].Revisions {
			if _, ok := s.cache[d]; !ok {
				need[d] = true
			}
		}
	}
	s.mu.Unlock()

	digests := make([]string, 0, len(need))
	for d := range need {
		digests = append(digests, d)
	}
	sort.Strings(digests)
	failed := map[string]string{}
	for start := 0; start < len(digests); start += readBatch {
		batch := digests[start:min(start+readBatch, len(digests))]
		read, err := s.read(ctx, batch)
		s.mu.Lock()
		for _, d := range batch {
			switch m := read[d]; {
			case err != nil:
				failed[d] = err.Error()
			case m == nil:
				failed[d] = "the store holds no manifest content for it"
			default:
				s.cache[d] = m
			}
		}
		s.mu.Unlock()
	}

	got := map[string]*Manifest{}
	unread := map[string]string{}
	s.mu.Lock()
	defer s.mu.Unlock()
	for _, name := range l.RepoNames() {
		for d := range l.Repos[name].Revisions {
			if m, ok := s.cache[d]; ok {
				got[Key(name, d)] = m
			} else if why, ok := failed[d]; ok {
				unread[Key(name, d)] = why
			} else {
				unread[Key(name, d)] = "not read"
			}
		}
	}
	return got, unread
}

// read reads one batch of manifests. A blob that is not a manifest, or not there, is answered as nil;
// an exec that fails fails the batch.
func (s *Files) read(ctx context.Context, digests []string) (map[string]*Manifest, error) {
	args := []string{"exec", s.Container, "sh", "-c", readScript, "sh"}
	byPath := map[string]string{}
	for _, d := range digests {
		if p := blobPath(d); p != "" {
			byPath[p] = d
			args = append(args, p)
		}
	}
	out, err := docker(ctx, s.Run, s.UID, args...)
	if err != nil {
		return nil, err
	}
	return parseManifests(out, byPath)
}

// parseManifests reads readScript's output.
func parseManifests(out string, byPath map[string]string) (map[string]*Manifest, error) {
	got := map[string]*Manifest{}
	current := ""
	var body strings.Builder
	flush := func() {
		if d := byPath[current]; d != "" {
			m, err := ParseManifest([]byte(body.String()))
			if err != nil {
				m = nil
			}
			got[d] = m
		}
		current = ""
		body.Reset()
	}
	ended := false
	for _, line := range strings.Split(out, "\n") {
		switch {
		case strings.HasPrefix(line, "#manifest "):
			flush()
			current = strings.TrimPrefix(line, "#manifest ")
		case line == "#absent":
			if d := byPath[current]; d != "" {
				got[d] = nil
			}
			current = ""
			body.Reset()
		case line == "#end":
			flush()
			ended = true
		default:
			body.WriteString(line)
			body.WriteString("\n")
		}
	}
	if !ended {
		return nil, fmt.Errorf("reading the store's manifests was cut short (it never reached its end)")
	}
	return got, nil
}

// Ran is what a command did: its output, its exit status, and why it never ran to an answer.
type Ran struct {
	Stdout string
	Stderr string
	Status int
	// Err is "ENOENT" when the program is not installed, or says it was ended for taking too long.
	Err string
}

// Runner runs one command, so every tool can be tested without a daemon.
type Runner func(ctx context.Context, name string, args ...string) Ran

// ListTimeout is how long listing the store's files may take: below the runtime's thirty-second call
// limit, so a store that hangs is answered as such rather than as a call the runtime gave up on.
const ListTimeout = 15 * time.Second

// ExecRunner runs a command on this machine, bounded by ListTimeout.
func ExecRunner(ctx context.Context, name string, args ...string) Ran {
	ctx, cancel := context.WithTimeout(ctx, ListTimeout)
	defer cancel()
	cmd := exec.CommandContext(ctx, name, args...)
	var out, errb bytes.Buffer
	cmd.Stdout, cmd.Stderr = &out, &errb
	err := cmd.Run()
	r := Ran{Stdout: out.String(), Stderr: errb.String()}
	var exitErr *exec.ExitError
	switch {
	case errors.Is(ctx.Err(), context.DeadlineExceeded):
		r.Status, r.Err = 124, fmt.Sprintf("no answer within %d s", int(ListTimeout/time.Second))
	case errors.Is(err, exec.ErrNotFound):
		r.Status, r.Err = 127, "ENOENT"
	case errors.As(err, &exitErr):
		r.Status = exitErr.ExitCode()
	case err != nil:
		r.Status, r.Err = 1, err.Error()
	}
	return r
}

var socketRefused = regexp.MustCompile(`(?i)permission denied.*docker.*sock|docker\.sock.*permission denied`)

// docker runs one docker command as this account and answers its stdout. A socket that refuses the
// account is asked again through `sudo -n`, as the container runtime's own tools do; never as root
// otherwise, and never with a prompt.
func docker(ctx context.Context, run Runner, uid int, args ...string) (string, error) {
	r := run(ctx, "docker", args...)
	program := "docker"
	if r.Status != 0 && r.Err == "" && uid != 0 && socketRefused.MatchString(r.Stderr) {
		program = "sudo"
		r = run(ctx, "sudo", append([]string{"-n", "docker"}, args...)...)
	}
	if r.Status == 0 && r.Err == "" {
		return r.Stdout, nil
	}
	said := strings.TrimSpace(r.Stderr + "\n" + r.Stdout)
	switch {
	case r.Err == "ENOENT" && program == "sudo":
		return "", errors.New("the runtime's socket refused this account, and sudo is not installed here to escalate with")
	case r.Err == "ENOENT":
		return "", errors.New("docker is not installed on this machine, so the store's files cannot be listed")
	case r.Err != "":
		return "", fmt.Errorf("listing the store's files did not answer: %s", r.Err)
	case program == "sudo" && strings.HasPrefix(said, "sudo:"):
		return "", fmt.Errorf("the runtime's socket refused this account and it may not escalate without a prompt: %s", firstLine(said))
	case strings.Contains(said, "No such container"):
		return "", fmt.Errorf("the store's container is not on this machine: %s", firstLine(said))
	case strings.Contains(said, "is not running"):
		return "", fmt.Errorf("the store's container is not running: %s", firstLine(said))
	}
	if l := firstLine(said); l != "" {
		return "", fmt.Errorf("listing the store's files failed (%d): %s", r.Status, l)
	}
	return "", fmt.Errorf("listing the store's files failed with status %d", r.Status)
}

func firstLine(s string) string {
	for _, l := range strings.Split(s, "\n") {
		if l = strings.TrimSpace(l); l != "" {
			return l
		}
	}
	return ""
}