gitea: let an agent put a commit on a branch of its own, and read history, compares, reviews, directories and code (issue 375)

An agent could not get a finished change onto the forge without someone
pushing by hand, so the branch / pull request / approval rule could not be
followed. A Go bundle serves gitea_commit, held to gitea_api's rule (never the
default branch, never a protected one), and the summarised reads agents
otherwise made through gitea_api as raw JSON.

modules/gitea/cmd/forge/search.go

@@ -0,0 +1,171 @@
+package main
+
+import (
+	"archive/tar"
+	"bufio"
+	"bytes"
+	"compress/gzip"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"regexp"
+	"strings"
+)
+
+// gitea_search_code: the forge's API has no code search, so the repository's archive at a ref is read once
+// and searched here (novox/hq issue 375). One request, nothing kept.
+
+// Match is one line that matched.
+type Match struct {
+	Path string `json:"path"`
+	Line int    `json:"line"`
+	Text string `json:"text"`
+}
+
+// Found is a search's answer.
+type Found struct {
+	Repository string   `json:"repository"`
+	Ref        string   `json:"ref"`
+	Query      string   `json:"query"`
+	Matches    []Match  `json:"matches"`
+	Files      int      `json:"files_searched"`
+	Skipped    []string `json:"skipped,omitempty"`
+	Truncated  bool     `json:"truncated"`
+	Said       []string `json:"said"`
+}
+
+// SearchAsk is one gitea_search_code call.
+type SearchAsk struct {
+	Owner, Repo, Ref, Query, Path string
+	Regex, IgnoreCase             bool
+	Max                           int
+}
+
+const (
+	maxFileSearched = 4 << 20 // a larger file is skipped and named
+	maxLineShown    = 300
+)
+
+func searchCode(ctx context.Context, f *Forge, a SearchAsk) (*Found, error) {
+	repo, err := repoPath(a.Owner, a.Repo)
+	if err != nil {
+		return nil, err
+	}
+	if a.Query == "" {
+		return nil, refuse("query is required. Nothing was sent")
+	}
+	pattern := a.Query
+	if !a.Regex {
+		pattern = regexp.QuoteMeta(pattern)
+	}
+	if a.IgnoreCase {
+		pattern = "(?i)" + pattern
+	}
+	re, err := regexp.Compile(pattern)
+	if err != nil {
+		return nil, refuse("query is not a regular expression Go reads: %v. Nothing was sent", err)
+	}
+	ref := a.Ref
+	if ref == "" {
+		var r Repo
+		if err := f.getJSON(ctx, repo, &r); err != nil {
+			return nil, err
+		}
+		if r.Empty || r.DefaultBranch == "" {
+			return &Found{Repository: a.Owner + "/" + a.Repo, Query: a.Query, Matches: []Match{}, Said: []string{"the repository is empty"}}, nil
+		}
+		ref = r.DefaultBranch
+	}
+	seg, err := segments("ref", ref)
+	if err != nil {
+		return nil, err
+	}
+	prefix := strings.Trim(a.Path, "/")
+	if prefix != "" {
+		if _, err := segments("path", prefix); err != nil {
+			return nil, err
+		}
+	}
+	body, err := f.do(ctx, "GET", repo+"/archive/"+seg+".tar.gz", nil, maxArchive)
+	if err != nil {
+		if errors.Is(err, errNotFound) {
+			return nil, refuse("%s/%s has no ref %s", a.Owner, a.Repo, ref)
+		}
+		return nil, err
+	}
+	out := &Found{Repository: a.Owner + "/" + a.Repo, Ref: ref, Query: a.Query, Matches: []Match{}}
+	if err := searchArchive(body, prefix, re, a.Max, out); err != nil {
+		return nil, fmt.Errorf("the archive of %s at %s could not be read: %w", out.Repository, ref, err)
+	}
+	said := fmt.Sprintf("%d matches in %d files searched", len(out.Matches), out.Files)
+	if out.Truncated {
+		said = fmt.Sprintf("the first %d matches, and more were found: narrow with path or the query", len(out.Matches))
+	}
+	out.Said = append(out.Said, said)
+	if len(out.Skipped) > 0 {
+		out.Said = append(out.Said, fmt.Sprintf("%d files larger than %d MiB were not searched: they are under skipped", len(out.Skipped), maxFileSearched>>20))
+	}
+	return out, nil
+}
+
+// searchArchive reads a gzipped tar the forge made — every path under one top directory — and collects the
+// lines that match, from text files under the prefix.
+func searchArchive(gz []byte, prefix string, re *regexp.Regexp, max int, out *Found) error {
+	zr, err := gzip.NewReader(bytes.NewReader(gz))
+	if err != nil {
+		return err
+	}
+	tr := tar.NewReader(zr)
+	for {
+		h, err := tr.Next()
+		if err == io.EOF {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+		if h.Typeflag != tar.TypeReg {
+			continue
+		}
+		_, path, found := strings.Cut(h.Name, "/")
+		if !found || path == "" {
+			continue
+		}
+		if prefix != "" && path != prefix && !strings.HasPrefix(path, prefix+"/") {
+			continue
+		}
+		if h.Size > maxFileSearched {
+			out.Skipped = append(out.Skipped, path)
+			continue
+		}
+		content, err := io.ReadAll(io.LimitReader(tr, maxFileSearched+1))
+		if err != nil {
+			return err
+		}
+		head := content
+		if len(head) > 8000 {
+			head = head[:8000]
+		}
+		if bytes.IndexByte(head, 0) >= 0 {
+			continue // binary
+		}
+		out.Files++
+		sc := bufio.NewScanner(bytes.NewReader(content))
+		sc.Buffer(make([]byte, 64<<10), maxFileSearched+1)
+		for n := 1; sc.Scan(); n++ {
+			line := sc.Text()
+			if !re.MatchString(line) {
+				continue
+			}
+			if len(out.Matches) >= max {
+				out.Truncated = true
+				return nil
+			}
+			if len(line) > maxLineShown {
+				line = strings.ToValidUTF8(line[:maxLineShown], "") + "…"
+			}
+			out.Matches = append(out.Matches, Match{Path: path, Line: n, Text: line})
+		}
+	}
+}