package main import ( "archive/tar" "bufio" "bytes" "compress/gzip" "context" "errors" "fmt" "io" "regexp" "strings" ) // gitea_search_code: the forge's API has no code search, so the repository's archive at a ref is read once // and searched here (novox/hq issue 375). One request, nothing kept. // Match is one line that matched. type Match struct { Path string `json:"path"` Line int `json:"line"` Text string `json:"text"` } // Found is a search's answer. type Found struct { Repository string `json:"repository"` Ref string `json:"ref"` Query string `json:"query"` Matches []Match `json:"matches"` Files int `json:"files_searched"` Skipped []string `json:"skipped,omitempty"` Truncated bool `json:"truncated"` Said []string `json:"said"` } // SearchAsk is one gitea_search_code call. type SearchAsk struct { Owner, Repo, Ref, Query, Path string Regex, IgnoreCase bool Max int } const ( maxFileSearched = 4 << 20 // a larger file is skipped and named maxLineShown = 300 ) func searchCode(ctx context.Context, f *Forge, a SearchAsk) (*Found, error) { repo, err := repoPath(a.Owner, a.Repo) if err != nil { return nil, err } if a.Query == "" { return nil, refuse("query is required. Nothing was sent") } pattern := a.Query if !a.Regex { pattern = regexp.QuoteMeta(pattern) } if a.IgnoreCase { pattern = "(?i)" + pattern } re, err := regexp.Compile(pattern) if err != nil { return nil, refuse("query is not a regular expression Go reads: %v. Nothing was sent", err) } ref := a.Ref if ref == "" { var r Repo if err := f.getJSON(ctx, repo, &r); err != nil { return nil, err } if r.Empty || r.DefaultBranch == "" { return &Found{Repository: a.Owner + "/" + a.Repo, Query: a.Query, Matches: []Match{}, Said: []string{"the repository is empty"}}, nil } ref = r.DefaultBranch } seg, err := segments("ref", ref) if err != nil { return nil, err } prefix := strings.Trim(a.Path, "/") if prefix != "" { if _, err := segments("path", prefix); err != nil { return nil, err } } body, err := f.do(ctx, "GET", repo+"/archive/"+seg+".tar.gz", nil, maxArchive) if err != nil { if errors.Is(err, errNotFound) { return nil, refuse("%s/%s has no ref %s", a.Owner, a.Repo, ref) } return nil, err } out := &Found{Repository: a.Owner + "/" + a.Repo, Ref: ref, Query: a.Query, Matches: []Match{}} if err := searchArchive(body, prefix, re, a.Max, out); err != nil { return nil, fmt.Errorf("the archive of %s at %s could not be read: %w", out.Repository, ref, err) } said := fmt.Sprintf("%d matches in %d files searched", len(out.Matches), out.Files) if out.Truncated { said = fmt.Sprintf("the first %d matches, and more were found: narrow with path or the query", len(out.Matches)) } out.Said = append(out.Said, said) if len(out.Skipped) > 0 { out.Said = append(out.Said, fmt.Sprintf("%d files larger than %d MiB were not searched: they are under skipped", len(out.Skipped), maxFileSearched>>20)) } return out, nil } // searchArchive reads a gzipped tar the forge made — every path under one top directory — and collects the // lines that match, from text files under the prefix. func searchArchive(gz []byte, prefix string, re *regexp.Regexp, max int, out *Found) error { zr, err := gzip.NewReader(bytes.NewReader(gz)) if err != nil { return err } tr := tar.NewReader(zr) for { h, err := tr.Next() if err == io.EOF { return nil } if err != nil { return err } if h.Typeflag != tar.TypeReg { continue } _, path, found := strings.Cut(h.Name, "/") if !found || path == "" { continue } if prefix != "" && path != prefix && !strings.HasPrefix(path, prefix+"/") { continue } if h.Size > maxFileSearched { out.Skipped = append(out.Skipped, path) continue } content, err := io.ReadAll(io.LimitReader(tr, maxFileSearched+1)) if err != nil { return err } head := content if len(head) > 8000 { head = head[:8000] } if bytes.IndexByte(head, 0) >= 0 { continue // binary } out.Files++ sc := bufio.NewScanner(bytes.NewReader(content)) sc.Buffer(make([]byte, 64<<10), maxFileSearched+1) for n := 1; sc.Scan(); n++ { line := sc.Text() if !re.MatchString(line) { continue } if len(out.Matches) >= max { out.Truncated = true return nil } if len(line) > maxLineShown { line = strings.ToValidUTF8(line[:maxLineShown], "") + "…" } out.Matches = append(out.Matches, Match{Path: path, Line: n, Text: line}) } } }