package main import ( "context" "encoding/base64" "errors" "fmt" "regexp" "sort" "strings" ) // copy_from (novox/hq issue 1365): a file or a directory of another repository of the forge, copied by the // module itself, so a large move between repositories is not typed through the caller. Read through the // forge's API as the module's reads are (the contents of a path at a commit), every file at one commit sha // resolved once; the commit carries the files as it carries any others, within the same limits. // CopyFrom is where a files entry takes its content from. type CopyFrom struct { Owner, Repo, Ref, Path string } // CopiedFrom says, for one copy_from entry, what was read and at which commit. type CopiedFrom struct { Path string `json:"path"` From string `json:"from"` Source string `json:"source_commit"` Files int `json:"files"` Bytes int `json:"bytes"` } var fullSHA = regexp.MustCompile(`^[0-9a-f]{40}$`) // readCopyFrom reads one files[i].copy_from. func readCopyFrom(i int, v any) (*CopyFrom, error) { m, ok := v.(map[string]any) if !ok { return nil, refuse("files[%d].copy_from is an object {owner, repo, ref, path}. Nothing was sent", i) } c := &CopyFrom{Owner: text(m, "owner"), Repo: text(m, "repo"), Ref: text(m, "ref"), Path: strings.Trim(text(m, "path"), "/")} if c.Owner == "" || c.Repo == "" || c.Ref == "" || c.Path == "" { return nil, refuse("files[%d].copy_from needs owner, repo, ref (a branch, tag or commit) and path. Nothing was sent", i) } if _, err := repoPath(c.Owner, c.Repo); err != nil { return nil, err } if _, err := segments("copy_from path", c.Path); err != nil { return nil, err } return c, nil } // copySource is one file to copy: where it goes, its content (base64 as the forge answered, once read) and size. type copySource struct { target, source string size int } // resolveRef is the full commit sha a ref names in a repository, "" when there is none. func (f *Forge) resolveRef(ctx context.Context, repo, ref string) (string, error) { if fullSHA.MatchString(ref) { return ref, nil } if _, err := segments("copy_from ref", ref); err != nil { return "", err } b, err := f.branch(ctx, repo, ref) if err != nil { return "", err } if b != nil && b.Commit.ID != "" { return b.Commit.ID, nil } // Not a branch: a tag or a short commit, as the commit history reads it. var got []forgeCommit if err := f.GetJSON(ctx, repo+"/commits?sha="+queryEscape(ref)+"&limit=1&stat=false&verification=false&files=false", &got); err != nil { if errors.Is(err, errNotFound) { return "", nil } return "", err } if len(got) == 0 { return "", nil } return got[0].SHA, nil } // contentEntry is one entry of the forge's answer for a path: a file or a symlink, or the entries of a directory. type contentEntry struct { Name string `json:"name"` Path string `json:"path"` Type string `json:"type"` Size int64 `json:"size"` Encoding string `json:"encoding"` Content string `json:"content"` } // contentsAt reads a path at a commit: a single entry for a file, a list for a directory; found false for none. func (f *Forge) contentsAt(ctx context.Context, repo, path, sha string) (one contentEntry, list []contentEntry, found bool, err error) { seg, err := segments("copy_from path", path) if err != nil { return one, nil, false, err } body, err := f.Do(ctx, "GET", repo+"/contents/"+seg+"?ref="+queryEscape(sha), nil, maxBody) if err != nil { if errors.Is(err, errNotFound) { return one, nil, false, nil } return one, nil, false, err } if jsonInto(body, &list) == nil { return one, list, true, nil } if err := jsonInto(body, &one); err != nil { return one, nil, false, fmt.Errorf("the forge's answer about %s is not readable: %w", path, err) } return one, nil, true, nil } // walkCopy lists every file under a directory at a commit, recursively; a symlink or anything not a file is refused. func (f *Forge) walkCopy(ctx context.Context, repo, label string, dir string, list []contentEntry, out *[]copySource, prefix, target string) error { for _, e := range list { p := e.Path if p == "" { p = dir + "/" + e.Name } switch e.Type { case "file": rel := strings.TrimPrefix(strings.TrimPrefix(p, prefix), "/") *out = append(*out, copySource{target: target + "/" + rel, source: p, size: int(e.Size)}) case "dir": _, sub, found, err := f.contentsAt(ctx, repo, p, label) if err != nil { return err } if !found { return fmt.Errorf("the forge listed %s but then did not have it", p) } if err := f.walkCopy(ctx, repo, label, p, sub, out, prefix, target); err != nil { return err } case "symlink": return refuse("%s is a symlink: copy_from copies files only. Nothing was sent", p) default: return refuse("%s is a %s: copy_from copies files only. Nothing was sent", p, e.Type) } } return nil } // expandCopies replaces every copy_from entry by the files it stands for, read at one commit per entry, and says // what was read. The limits count the copied files with the others before anything is read in full. func expandCopies(ctx context.Context, f *Forge, a CommitAsk) ([]FileChange, []CopiedFrom, error) { type plan struct { at int sha string sources []copySource from *CopyFrom repo string } var plans []plan count, total := 0, 0 for i, fc := range a.Files { if fc.CopyFrom == nil { count++ total += fc.size continue } c := fc.CopyFrom repo, err := repoPath(c.Owner, c.Repo) if err != nil { return nil, nil, err } var r Repo if err := f.GetJSON(ctx, repo, &r); err != nil { if errors.Is(err, errNotFound) { return nil, nil, refuse("files[%d].copy_from: the forge has no repository %s/%s. Nothing was sent", i, c.Owner, c.Repo) } return nil, nil, err } sha, err := f.resolveRef(ctx, repo, c.Ref) if err != nil { return nil, nil, err } if sha == "" { return nil, nil, refuse("files[%d].copy_from: %s/%s has no branch, tag or commit %q. Nothing was sent", i, c.Owner, c.Repo, c.Ref) } one, list, found, err := f.contentsAt(ctx, repo, c.Path, sha) if err != nil { return nil, nil, err } if !found { return nil, nil, refuse("files[%d].copy_from: %s/%s has no %q at %s. Nothing was sent", i, c.Owner, c.Repo, c.Path, short(sha)) } p := plan{at: i, sha: sha, from: c, repo: repo} if list != nil || one.Type == "dir" { if err := f.walkCopy(ctx, repo, sha, c.Path, list, &p.sources, c.Path, fc.Path); err != nil { return nil, nil, err } } else if one.Type == "file" { p.sources = []copySource{{target: fc.Path, source: c.Path, size: int(one.Size)}} } else { return nil, nil, refuse("files[%d].copy_from: %s is a %s, not a file or a directory: copy_from copies files only. Nothing was sent", i, c.Path, one.Type) } if len(p.sources) == 0 { return nil, nil, refuse("files[%d].copy_from: %s/%s has no file under %q at %s. Nothing was sent", i, c.Owner, c.Repo, c.Path, short(sha)) } for _, s := range p.sources { count++ total += s.size if s.size > maxFileBytes { return nil, nil, refuse("files[%d].copy_from: %s is %s, more than the %s one file of a commit may be. Nothing was sent", i, s.source, mib(s.size), mib(maxFileBytes)) } } plans = append(plans, p) } if count > maxFiles || total > maxCommitBytes { return nil, nil, refuse("the commit would carry %d files, %s together (copied and given), more than the %d files and %s one commit carries: "+ "split the copy by subdirectory into several commits. Nothing was sent", count, mib(total), maxFiles, mib(maxCommitBytes)) } // The files are read now, each at its commit sha. var out []FileChange var said []CopiedFrom seen := map[string]bool{} for _, fc := range a.Files { if fc.CopyFrom == nil { if seen[fc.Path] { return nil, nil, refuse("%s is named twice in files. Nothing was sent", fc.Path) } seen[fc.Path] = true out = append(out, fc) } } for _, p := range plans { cf := CopiedFrom{Path: a.Files[p.at].Path, From: p.from.Owner + "/" + p.from.Repo + ":" + p.from.Path, Source: p.sha} sort.Slice(p.sources, func(i, j int) bool { return p.sources[i].target < p.sources[j].target }) for _, s := range p.sources { if seen[s.target] { return nil, nil, refuse("%s is named twice in files (a copy_from reaches it as well). Nothing was sent", s.target) } seen[s.target] = true if _, err := segments("file path", s.target); err != nil { return nil, nil, err } one, _, found, err := f.contentsAt(ctx, p.repo, s.source, p.sha) if err != nil { return nil, nil, err } if !found || one.Type != "file" { return nil, nil, refuse("%s/%s: %s is not a file at %s any more. Nothing was sent", p.from.Owner, p.from.Repo, s.source, short(p.sha)) } if one.Encoding != "base64" { return nil, nil, fmt.Errorf("the forge answered %s without its content in base64 (encoding %q): nothing was sent", s.source, one.Encoding) } b64 := strings.NewReplacer("\n", "", "\r", "").Replace(one.Content) raw, err := base64.StdEncoding.DecodeString(b64) if err != nil { return nil, nil, fmt.Errorf("the forge's content of %s is not base64: %w", s.source, err) } if len(raw) > maxFileBytes { return nil, nil, refuse("%s is %s, more than the %s one file of a commit may be. Nothing was sent", s.source, mib(len(raw)), mib(maxFileBytes)) } cf.Files++ cf.Bytes += len(raw) out = append(out, FileChange{Path: s.target, ContentBase64: &b64, size: len(raw)}) } said = append(said, cf) } return out, said, nil }