mesh-catalog/modules/gitea/cmd/forge/copy.go

package main

import (
	"context"
	"encoding/base64"
	"errors"
	"fmt"
	"regexp"
	"sort"
	"strings"
)

// copy_from (novox/hq issue 1365): a file or a directory of another repository of the forge, copied by the
// module itself, so a large move between repositories is not typed through the caller. Read through the
// forge's API as the module's reads are (the contents of a path at a commit), every file at one commit sha
// resolved once; the commit carries the files as it carries any others, within the same limits.

// CopyFrom is where a files entry takes its content from.
type CopyFrom struct {
	Owner, Repo, Ref, Path string
}

// CopiedFrom says, for one copy_from entry, what was read and at which commit.
type CopiedFrom struct {
	Path   string `json:"path"`
	From   string `json:"from"`
	Source string `json:"source_commit"`
	Files  int    `json:"files"`
	Bytes  int    `json:"bytes"`
}

var fullSHA = regexp.MustCompile(`^[0-9a-f]{40}$`)

// readCopyFrom reads one files[i].copy_from.
func readCopyFrom(i int, v any) (*CopyFrom, error) {
	m, ok := v.(map[string]any)
	if !ok {
		return nil, refuse("files[%d].copy_from is an object {owner, repo, ref, path}. Nothing was sent", i)
	}
	c := &CopyFrom{Owner: text(m, "owner"), Repo: text(m, "repo"), Ref: text(m, "ref"), Path: strings.Trim(text(m, "path"), "/")}
	if c.Owner == "" || c.Repo == "" || c.Ref == "" || c.Path == "" {
		return nil, refuse("files[%d].copy_from needs owner, repo, ref (a branch, tag or commit) and path. Nothing was sent", i)
	}
	if _, err := repoPath(c.Owner, c.Repo); err != nil {
		return nil, err
	}
	if _, err := segments("copy_from path", c.Path); err != nil {
		return nil, err
	}
	return c, nil
}

// copySource is one file to copy: where it goes, its content (base64 as the forge answered, once read) and size.
type copySource struct {
	target, source string
	size           int
}

// resolveRef is the full commit sha a ref names in a repository, "" when there is none.
func (f *Forge) resolveRef(ctx context.Context, repo, ref string) (string, error) {
	if fullSHA.MatchString(ref) {
		return ref, nil
	}
	if _, err := segments("copy_from ref", ref); err != nil {
		return "", err
	}
	b, err := f.branch(ctx, repo, ref)
	if err != nil {
		return "", err
	}
	if b != nil && b.Commit.ID != "" {
		return b.Commit.ID, nil
	}
	// Not a branch: a tag or a short commit, as the commit history reads it.
	var got []forgeCommit
	if err := f.GetJSON(ctx, repo+"/commits?sha="+queryEscape(ref)+"&limit=1&stat=false&verification=false&files=false", &got); err != nil {
		if errors.Is(err, errNotFound) {
			return "", nil
		}
		return "", err
	}
	if len(got) == 0 {
		return "", nil
	}
	return got[0].SHA, nil
}

// contentEntry is one entry of the forge's answer for a path: a file or a symlink, or the entries of a directory.
type contentEntry struct {
	Name     string `json:"name"`
	Path     string `json:"path"`
	Type     string `json:"type"`
	Size     int64  `json:"size"`
	Encoding string `json:"encoding"`
	Content  string `json:"content"`
}

// contentsAt reads a path at a commit: a single entry for a file, a list for a directory; found false for none.
func (f *Forge) contentsAt(ctx context.Context, repo, path, sha string) (one contentEntry, list []contentEntry, found bool, err error) {
	seg, err := segments("copy_from path", path)
	if err != nil {
		return one, nil, false, err
	}
	body, err := f.Do(ctx, "GET", repo+"/contents/"+seg+"?ref="+queryEscape(sha), nil, maxBody)
	if err != nil {
		if errors.Is(err, errNotFound) {
			return one, nil, false, nil
		}
		return one, nil, false, err
	}
	if jsonInto(body, &list) == nil {
		return one, list, true, nil
	}
	if err := jsonInto(body, &one); err != nil {
		return one, nil, false, fmt.Errorf("the forge's answer about %s is not readable: %w", path, err)
	}
	return one, nil, true, nil
}

// walkCopy lists every file under a directory at a commit, recursively; a symlink or anything not a file is refused.
func (f *Forge) walkCopy(ctx context.Context, repo, label string, dir string, list []contentEntry, out *[]copySource, prefix, target string) error {
	for _, e := range list {
		p := e.Path
		if p == "" {
			p = dir + "/" + e.Name
		}
		switch e.Type {
		case "file":
			rel := strings.TrimPrefix(strings.TrimPrefix(p, prefix), "/")
			*out = append(*out, copySource{target: target + "/" + rel, source: p, size: int(e.Size)})
		case "dir":
			_, sub, found, err := f.contentsAt(ctx, repo, p, label)
			if err != nil {
				return err
			}
			if !found {
				return fmt.Errorf("the forge listed %s but then did not have it", p)
			}
			if err := f.walkCopy(ctx, repo, label, p, sub, out, prefix, target); err != nil {
				return err
			}
		case "symlink":
			return refuse("%s is a symlink: copy_from copies files only. Nothing was sent", p)
		default:
			return refuse("%s is a %s: copy_from copies files only. Nothing was sent", p, e.Type)
		}
	}
	return nil
}

// expandCopies replaces every copy_from entry by the files it stands for, read at one commit per entry, and says
// what was read. The limits count the copied files with the others before anything is read in full.
func expandCopies(ctx context.Context, f *Forge, a CommitAsk) ([]FileChange, []CopiedFrom, error) {
	type plan struct {
		at      int
		sha     string
		sources []copySource
		from    *CopyFrom
		repo    string
	}
	var plans []plan
	count, total := 0, 0
	for i, fc := range a.Files {
		if fc.CopyFrom == nil {
			count++
			total += fc.size
			continue
		}
		c := fc.CopyFrom
		repo, err := repoPath(c.Owner, c.Repo)
		if err != nil {
			return nil, nil, err
		}
		var r Repo
		if err := f.GetJSON(ctx, repo, &r); err != nil {
			if errors.Is(err, errNotFound) {
				return nil, nil, refuse("files[%d].copy_from: the forge has no repository %s/%s. Nothing was sent", i, c.Owner, c.Repo)
			}
			return nil, nil, err
		}
		sha, err := f.resolveRef(ctx, repo, c.Ref)
		if err != nil {
			return nil, nil, err
		}
		if sha == "" {
			return nil, nil, refuse("files[%d].copy_from: %s/%s has no branch, tag or commit %q. Nothing was sent", i, c.Owner, c.Repo, c.Ref)
		}
		one, list, found, err := f.contentsAt(ctx, repo, c.Path, sha)
		if err != nil {
			return nil, nil, err
		}
		if !found {
			return nil, nil, refuse("files[%d].copy_from: %s/%s has no %q at %s. Nothing was sent", i, c.Owner, c.Repo, c.Path, short(sha))
		}
		p := plan{at: i, sha: sha, from: c, repo: repo}
		if list != nil || one.Type == "dir" {
			if err := f.walkCopy(ctx, repo, sha, c.Path, list, &p.sources, c.Path, fc.Path); err != nil {
				return nil, nil, err
			}
		} else if one.Type == "file" {
			p.sources = []copySource{{target: fc.Path, source: c.Path, size: int(one.Size)}}
		} else {
			return nil, nil, refuse("files[%d].copy_from: %s is a %s, not a file or a directory: copy_from copies files only. Nothing was sent", i, c.Path, one.Type)
		}
		if len(p.sources) == 0 {
			return nil, nil, refuse("files[%d].copy_from: %s/%s has no file under %q at %s. Nothing was sent", i, c.Owner, c.Repo, c.Path, short(sha))
		}
		for _, s := range p.sources {
			count++
			total += s.size
			if s.size > maxFileBytes {
				return nil, nil, refuse("files[%d].copy_from: %s is %s, more than the %s one file of a commit may be. Nothing was sent", i, s.source, mib(s.size), mib(maxFileBytes))
			}
		}
		plans = append(plans, p)
	}
	if count > maxFiles || total > maxCommitBytes {
		return nil, nil, refuse("the commit would carry %d files, %s together (copied and given), more than the %d files and %s one commit carries: "+
			"split the copy by subdirectory into several commits. Nothing was sent", count, mib(total), maxFiles, mib(maxCommitBytes))
	}
	// The files are read now, each at its commit sha.
	var out []FileChange
	var said []CopiedFrom
	seen := map[string]bool{}
	for _, fc := range a.Files {
		if fc.CopyFrom == nil {
			if seen[fc.Path] {
				return nil, nil, refuse("%s is named twice in files. Nothing was sent", fc.Path)
			}
			seen[fc.Path] = true
			out = append(out, fc)
		}
	}
	for _, p := range plans {
		cf := CopiedFrom{Path: a.Files[p.at].Path, From: p.from.Owner + "/" + p.from.Repo + ":" + p.from.Path, Source: p.sha}
		sort.Slice(p.sources, func(i, j int) bool { return p.sources[i].target < p.sources[j].target })
		for _, s := range p.sources {
			if seen[s.target] {
				return nil, nil, refuse("%s is named twice in files (a copy_from reaches it as well). Nothing was sent", s.target)
			}
			seen[s.target] = true
			if _, err := segments("file path", s.target); err != nil {
				return nil, nil, err
			}
			one, _, found, err := f.contentsAt(ctx, p.repo, s.source, p.sha)
			if err != nil {
				return nil, nil, err
			}
			if !found || one.Type != "file" {
				return nil, nil, refuse("%s/%s: %s is not a file at %s any more. Nothing was sent", p.from.Owner, p.from.Repo, s.source, short(p.sha))
			}
			if one.Encoding != "base64" {
				return nil, nil, fmt.Errorf("the forge answered %s without its content in base64 (encoding %q): nothing was sent", s.source, one.Encoding)
			}
			b64 := strings.NewReplacer("\n", "", "\r", "").Replace(one.Content)
			raw, err := base64.StdEncoding.DecodeString(b64)
			if err != nil {
				return nil, nil, fmt.Errorf("the forge's content of %s is not base64: %w", s.source, err)
			}
			if len(raw) > maxFileBytes {
				return nil, nil, refuse("%s is %s, more than the %s one file of a commit may be. Nothing was sent", s.source, mib(len(raw)), mib(maxFileBytes))
			}
			cf.Files++
			cf.Bytes += len(raw)
			out = append(out, FileChange{Path: s.target, ContentBase64: &b64, size: len(raw)})
		}
		said = append(said, cf)
	}
	return out, said, nil
}