package main import ( "archive/tar" "bufio" "bytes" "compress/gzip" "context" "errors" "fmt" "io" "regexp" "strings" ) // gitea_search_code: the forge's API has no code search, so the repository's archive at a ref is read once // and searched here (novox/hq issue 375). One request, nothing kept. // Match is one line that matched. type Match struct { Path string `json:"path"` Line int `json:"line"` Text string `json:"text"` } // Found is a search's answer. type Found struct { Repository string `json:"repository"` Ref string `json:"ref"` Query string `json:"query"` Matches []Match `json:"matches"` Files int `json:"files_searched"` Skipped []string `json:"skipped,omitempty"` Truncated bool `json:"truncated"` Stopped bool `json:"stopped_at_deadline,omitempty"` Said []string `json:"said"` } // SearchAsk is one gitea_search_code call. type SearchAsk struct { Owner, Repo, Ref, Query, Path string Regex, IgnoreCase bool Max int } const ( maxFileSearched = 4 << 20 // a larger file is skipped and named maxLineShown = 300 ) func searchCode(ctx context.Context, f *Forge, a SearchAsk) (*Found, error) { repo, err := repoPath(a.Owner, a.Repo) if err != nil { return nil, err } if a.Query == "" { return nil, refuse("query is required. Nothing was sent") } pattern := a.Query if !a.Regex { pattern = regexp.QuoteMeta(pattern) } if a.IgnoreCase { pattern = "(?i)" + pattern } re, err := regexp.Compile(pattern) if err != nil { return nil, refuse("query is not a regular expression Go reads: %v. Nothing was sent", err) } ref := a.Ref if ref == "" { var r Repo if err := f.GetJSON(ctx, repo, &r); err != nil { return nil, err } if r.Empty || r.DefaultBranch == "" { return &Found{Repository: a.Owner + "/" + a.Repo, Query: a.Query, Matches: []Match{}, Said: []string{"the repository is empty"}}, nil } ref = r.DefaultBranch } seg, err := segments("ref", ref) if err != nil { return nil, err } prefix := strings.Trim(a.Path, "/") if prefix != "" { if _, err := segments("path", prefix); err != nil { return nil, err } } body, err := f.Do(ctx, "GET", repo+"/archive/"+seg+".tar.gz", nil, maxArchive) if err != nil { if errors.Is(err, errNotFound) { return nil, refuse("%s/%s has no ref %s", a.Owner, a.Repo, ref) } return nil, err } out := &Found{Repository: a.Owner + "/" + a.Repo, Ref: ref, Query: a.Query, Matches: []Match{}} if err := searchArchive(ctx, body, prefix, re, a.Max, out); err != nil { return nil, fmt.Errorf("the archive of %s at %s could not be read: %w", out.Repository, ref, err) } said := fmt.Sprintf("%d matches in %d files searched", len(out.Matches), out.Files) switch { case out.Stopped: said = fmt.Sprintf("the search stopped at its deadline of %s after %d files, with %d matches: the rest of the "+ "archive was not searched. Narrow with path", deadline, out.Files, len(out.Matches)) case out.Truncated: said = fmt.Sprintf("the first %d matches, and more were found: narrow with path or the query", len(out.Matches)) } out.Said = append(out.Said, said) if len(out.Skipped) > 0 { out.Said = append(out.Said, fmt.Sprintf("%d files larger than %d MiB were not searched: they are under skipped", len(out.Skipped), maxFileSearched>>20)) } return out, nil } // searchArchive reads a gzipped tar the forge made — every path under one top directory — and collects the // lines that match, from text files under the prefix. It stops at the call's deadline (novox/hq issue 539): // what it found by then is kept, and out.Stopped says the rest was not searched. func searchArchive(ctx context.Context, gz []byte, prefix string, re *regexp.Regexp, max int, out *Found) error { zr, err := gzip.NewReader(bytes.NewReader(gz)) if err != nil { return err } tr := tar.NewReader(ctxReader{ctx, zr}) stopped := func(err error) bool { if ctx.Err() != nil && (err == nil || errors.Is(err, ctx.Err())) { out.Stopped = true return true } return false } for { if stopped(nil) { return nil } h, err := tr.Next() if err == io.EOF { return nil } if stopped(err) { return nil } if err != nil { return err } if h.Typeflag != tar.TypeReg { continue } _, path, found := strings.Cut(h.Name, "/") if !found || path == "" { continue } if prefix != "" && path != prefix && !strings.HasPrefix(path, prefix+"/") { continue } if h.Size > maxFileSearched { out.Skipped = append(out.Skipped, path) continue } content, err := io.ReadAll(io.LimitReader(tr, maxFileSearched+1)) if stopped(err) { return nil } if err != nil { return err } head := content if len(head) > 8000 { head = head[:8000] } if bytes.IndexByte(head, 0) >= 0 { continue // binary } out.Files++ sc := bufio.NewScanner(bytes.NewReader(content)) sc.Buffer(make([]byte, 64<<10), maxFileSearched+1) for n := 1; sc.Scan(); n++ { line := sc.Text() if !re.MatchString(line) { continue } if len(out.Matches) >= max { out.Truncated = true return nil } if len(line) > maxLineShown { line = strings.ToValidUTF8(line[:maxLineShown], "") + "…" } out.Matches = append(out.Matches, Match{Path: path, Line: n, Text: line}) } } } // ctxReader is a reader that ends when its context does: decompressing is the long part of a search. type ctxReader struct { ctx context.Context r io.Reader } func (c ctxReader) Read(p []byte) (int, error) { if err := c.ctx.Err(); err != nil { return 0, err } return c.r.Read(p) }