mesh-catalog/modules/gitea/cmd/forge/search.go

package main

import (
	"archive/tar"
	"bufio"
	"bytes"
	"compress/gzip"
	"context"
	"errors"
	"fmt"
	"io"
	"regexp"
	"strings"
)

// gitea_search_code: the forge's API has no code search, so the repository's archive at a ref is read once
// and searched here (novox/hq issue 375). One request, nothing kept.

// Match is one line that matched.
type Match struct {
	Path string `json:"path"`
	Line int    `json:"line"`
	Text string `json:"text"`
}

// Found is a search's answer.
type Found struct {
	Repository string   `json:"repository"`
	Ref        string   `json:"ref"`
	Query      string   `json:"query"`
	Matches    []Match  `json:"matches"`
	Files      int      `json:"files_searched"`
	Skipped    []string `json:"skipped,omitempty"`
	Truncated  bool     `json:"truncated"`
	Stopped    bool     `json:"stopped_at_deadline,omitempty"`
	Said       []string `json:"said"`
}

// SearchAsk is one gitea_search_code call.
type SearchAsk struct {
	Owner, Repo, Ref, Query, Path string
	Regex, IgnoreCase             bool
	Max                           int
}

const (
	maxFileSearched = 4 << 20 // a larger file is skipped and named
	maxLineShown    = 300
)

func searchCode(ctx context.Context, f *Forge, a SearchAsk) (*Found, error) {
	repo, err := repoPath(a.Owner, a.Repo)
	if err != nil {
		return nil, err
	}
	if a.Query == "" {
		return nil, refuse("query is required. Nothing was sent")
	}
	pattern := a.Query
	if !a.Regex {
		pattern = regexp.QuoteMeta(pattern)
	}
	if a.IgnoreCase {
		pattern = "(?i)" + pattern
	}
	re, err := regexp.Compile(pattern)
	if err != nil {
		return nil, refuse("query is not a regular expression Go reads: %v. Nothing was sent", err)
	}
	ref := a.Ref
	if ref == "" {
		var r Repo
		if err := f.GetJSON(ctx, repo, &r); err != nil {
			return nil, err
		}
		if r.Empty || r.DefaultBranch == "" {
			return &Found{Repository: a.Owner + "/" + a.Repo, Query: a.Query, Matches: []Match{}, Said: []string{"the repository is empty"}}, nil
		}
		ref = r.DefaultBranch
	}
	seg, err := segments("ref", ref)
	if err != nil {
		return nil, err
	}
	prefix := strings.Trim(a.Path, "/")
	if prefix != "" {
		if _, err := segments("path", prefix); err != nil {
			return nil, err
		}
	}
	body, err := f.Do(ctx, "GET", repo+"/archive/"+seg+".tar.gz", nil, maxArchive)
	if err != nil {
		if errors.Is(err, errNotFound) {
			return nil, refuse("%s/%s has no ref %s", a.Owner, a.Repo, ref)
		}
		return nil, err
	}
	out := &Found{Repository: a.Owner + "/" + a.Repo, Ref: ref, Query: a.Query, Matches: []Match{}}
	if err := searchArchive(ctx, body, prefix, re, a.Max, out); err != nil {
		return nil, fmt.Errorf("the archive of %s at %s could not be read: %w", out.Repository, ref, err)
	}
	said := fmt.Sprintf("%d matches in %d files searched", len(out.Matches), out.Files)
	switch {
	case out.Stopped:
		said = fmt.Sprintf("the search stopped at its deadline of %s after %d files, with %d matches: the rest of the "+
			"archive was not searched. Narrow with path", deadline, out.Files, len(out.Matches))
	case out.Truncated:
		said = fmt.Sprintf("the first %d matches, and more were found: narrow with path or the query", len(out.Matches))
	}
	out.Said = append(out.Said, said)
	if len(out.Skipped) > 0 {
		out.Said = append(out.Said, fmt.Sprintf("%d files larger than %d MiB were not searched: they are under skipped", len(out.Skipped), maxFileSearched>>20))
	}
	return out, nil
}

// searchArchive reads a gzipped tar the forge made — every path under one top directory — and collects the
// lines that match, from text files under the prefix. It stops at the call's deadline (novox/hq issue 539):
// what it found by then is kept, and out.Stopped says the rest was not searched.
func searchArchive(ctx context.Context, gz []byte, prefix string, re *regexp.Regexp, max int, out *Found) error {
	zr, err := gzip.NewReader(bytes.NewReader(gz))
	if err != nil {
		return err
	}
	tr := tar.NewReader(ctxReader{ctx, zr})
	stopped := func(err error) bool {
		if ctx.Err() != nil && (err == nil || errors.Is(err, ctx.Err())) {
			out.Stopped = true
			return true
		}
		return false
	}
	for {
		if stopped(nil) {
			return nil
		}
		h, err := tr.Next()
		if err == io.EOF {
			return nil
		}
		if stopped(err) {
			return nil
		}
		if err != nil {
			return err
		}
		if h.Typeflag != tar.TypeReg {
			continue
		}
		_, path, found := strings.Cut(h.Name, "/")
		if !found || path == "" {
			continue
		}
		if prefix != "" && path != prefix && !strings.HasPrefix(path, prefix+"/") {
			continue
		}
		if h.Size > maxFileSearched {
			out.Skipped = append(out.Skipped, path)
			continue
		}
		content, err := io.ReadAll(io.LimitReader(tr, maxFileSearched+1))
		if stopped(err) {
			return nil
		}
		if err != nil {
			return err
		}
		head := content
		if len(head) > 8000 {
			head = head[:8000]
		}
		if bytes.IndexByte(head, 0) >= 0 {
			continue // binary
		}
		out.Files++
		sc := bufio.NewScanner(bytes.NewReader(content))
		sc.Buffer(make([]byte, 64<<10), maxFileSearched+1)
		for n := 1; sc.Scan(); n++ {
			line := sc.Text()
			if !re.MatchString(line) {
				continue
			}
			if len(out.Matches) >= max {
				out.Truncated = true
				return nil
			}
			if len(line) > maxLineShown {
				line = strings.ToValidUTF8(line[:maxLineShown], "") + "…"
			}
			out.Matches = append(out.Matches, Match{Path: path, Line: n, Text: line})
		}
	}
}

// ctxReader is a reader that ends when its context does: decompressing is the long part of a search.
type ctxReader struct {
	ctx context.Context
	r   io.Reader
}

func (c ctxReader) Read(p []byte) (int, error) {
	if err := c.ctx.Err(); err != nil {
		return 0, err
	}
	return c.r.Read(p)
}