gitea forge bundle: name the parent the forge used, match protection rules as the forge does, bound a commit's size and a search's time (issue 539)

The forge's contents and diffpatch endpoints take a branch, never a commit to
build on, so the parent is read from the commit the forge made, and a branch
that moved under the commit is said. Blobs are read at the commit read first,
not at the branch name. Protection rules match as Gitea 1.27 matches them
([...], {a,b}, escapes, a plain name in any case), in the Go bundle and in
guard.ts alike. One commit carries at most 4 MiB per file or patch and 16 MiB
together; code search stops at its deadline and says so. The HTTP client the
two Go bundles copied is one package, internal/forgehttp.
This commit is contained in:
jochen
2026-10-12 12:51:22 +02:00
parent c5abdf35ed
commit f2d765320a
13 changed files with 724 additions and 314 deletions
+90 -16
View File
@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"net/http"
"net/url"
"strings"
"unicode/utf8"
)
@@ -52,8 +53,14 @@ type CommittedFile struct {
Operation string `json:"operation"`
}
// maxFiles is the most files one commit carries.
const maxFiles = 200
// What one commit carries at most (novox/hq issue 539): files, one file's content, one patch, and every
// file's content together, in bytes as committed (content_base64 decoded). Larger work is several commits.
const (
maxFiles = 200
maxFileBytes = 4 << 20
maxPatchBytes = 4 << 20
maxCommitBytes = 16 << 20
)
// readCommitAsk reads the tool's arguments.
func readCommitAsk(args map[string]any) (CommitAsk, error) {
@@ -109,8 +116,11 @@ func readCommitAsk(args map[string]any) (CommitAsk, error) {
return a, refuse("give files or patch, not both: one commit carries one of them. Nothing was sent")
case len(a.Files) > maxFiles:
return a, refuse("%d files is more than the %d one commit carries. Nothing was sent", len(a.Files), maxFiles)
case len(a.Patch) > maxPatchBytes:
return a, refuse("the patch is %s, more than the %s one commit carries: split it into several commits. Nothing was sent", mib(len(a.Patch)), mib(maxPatchBytes))
}
seen := map[string]bool{}
total := 0
for i, fc := range a.Files {
if strings.HasPrefix(fc.Path, "/") {
return a, refuse("files[%d].path %q is relative to the repository's root, without a leading slash. Nothing was sent", i, fc.Path)
@@ -134,15 +144,35 @@ func readCommitAsk(args map[string]any) (CommitAsk, error) {
if fc.Content != nil && !utf8.ValidString(*fc.Content) {
return a, refuse("files[%d].content (%s) is not UTF-8 text: give it as content_base64. Nothing was sent", i, fc.Path)
}
if fc.ContentBase64 != nil {
if _, err := base64.StdEncoding.DecodeString(*fc.ContentBase64); err != nil {
size := 0
switch {
case fc.Content != nil:
size = len(*fc.Content)
case fc.ContentBase64 != nil:
if base64.StdEncoding.DecodedLen(len(*fc.ContentBase64)) > maxFileBytes+3 {
size = base64.StdEncoding.DecodedLen(len(*fc.ContentBase64))
break
}
raw, err := base64.StdEncoding.DecodeString(*fc.ContentBase64)
if err != nil {
return a, refuse("files[%d].content_base64 (%s) is not base64: %v. Nothing was sent", i, fc.Path, err)
}
size = len(raw)
}
if size > maxFileBytes {
return a, refuse("files[%d] (%s) is %s, more than the %s one file of a commit may be. Nothing was sent", i, fc.Path, mib(size), mib(maxFileBytes))
}
total += size
}
if total > maxCommitBytes {
return a, refuse("the files are %s together, more than the %s one commit carries: split them into several commits. Nothing was sent", mib(total), mib(maxCommitBytes))
}
return a, nil
}
// mib says a size in bytes in MiB, to a tenth.
func mib(n int) string { return fmt.Sprintf("%.1f MiB", float64(n)/(1<<20)) }
// targetRefused says why a commit may not go on a branch: the default branch, or one a protection rule names.
func targetRefused(target, defaultBranch string, rules []Protection) error {
if target == defaultBranch {
@@ -172,7 +202,7 @@ func commit(ctx context.Context, f *Forge, a CommitAsk) (*Committed, error) {
return nil, err
}
var r Repo
if err := f.getJSON(ctx, repo, &r); err != nil {
if err := f.GetJSON(ctx, repo, &r); err != nil {
return nil, err
}
if r.DefaultBranch == "" {
@@ -180,7 +210,7 @@ func commit(ctx context.Context, f *Forge, a CommitAsk) (*Committed, error) {
}
// Read the rules, or refuse: a branch the rules could not be read for is never written (fails closed).
var rules []Protection
if err := f.getJSON(ctx, repo+"/branch_protections", &rules); err != nil {
if err := f.GetJSON(ctx, repo+"/branch_protections", &rules); err != nil {
return nil, fmt.Errorf("the protection rules of %s/%s could not be read, so the commit was not judged and nothing was sent: %w", a.Owner, a.Repo, err)
}
if err := targetRefused(a.Branch, r.DefaultBranch, rules); err != nil {
@@ -192,9 +222,13 @@ func commit(ctx context.Context, f *Forge, a CommitAsk) (*Committed, error) {
if err != nil {
return nil, err
}
ref := a.Branch // where the files are read and the commit goes
// The files are judged at the commit just read, never at a branch name that could move between two
// reads (novox/hq issue 539): a file the branch changes meanwhile is refused by the forge, whose update
// and delete carry the blob read here.
var ref string
if existing != nil {
out.Parent = existing.Commit.ID
ref = existing.Commit.ID
if a.Base != "" {
out.Said = append(out.Said, fmt.Sprintf("%s exists, at %s: base %s was not used", a.Branch, short(existing.Commit.ID), a.Base))
}
@@ -204,7 +238,6 @@ func commit(ctx context.Context, f *Forge, a CommitAsk) (*Committed, error) {
if out.Base == "" {
out.Base = r.DefaultBranch
}
ref = out.Base
b, err := f.branch(ctx, repo, out.Base)
if err != nil {
return nil, err
@@ -213,8 +246,6 @@ func commit(ctx context.Context, f *Forge, a CommitAsk) (*Committed, error) {
return nil, refuse("%s does not exist, and its base %s is not a branch of %s/%s. Nothing was sent", a.Branch, out.Base, a.Owner, a.Repo)
}
out.Parent = b.Commit.ID
// The files are judged at the commit just read; a file the base changes meanwhile is refused by the
// forge, whose update and delete carry the blob read here.
ref = b.Commit.ID
}
@@ -227,14 +258,14 @@ func commit(ctx context.Context, f *Forge, a CommitAsk) (*Committed, error) {
on["branch"] = a.Branch
}
// The forge takes no commit to build on, only a branch (Gitea 1.27: neither /contents nor /diffpatch
// carries a last commit), so the parent is read from what it made, never assumed (novox/hq issue 539).
var made struct {
Commit struct {
SHA string `json:"sha"`
} `json:"commit"`
Commit madeCommit `json:"commit"`
}
if a.Patch != "" {
on["content"] = a.Patch
if err := f.sendJSON(ctx, http.MethodPost, repo+"/diffpatch", on, &made); err != nil {
if err := f.SendJSON(ctx, http.MethodPost, repo+"/diffpatch", on, &made); err != nil {
return nil, fmt.Errorf("the forge did not take the patch: %w", err)
}
} else {
@@ -266,7 +297,7 @@ func commit(ctx context.Context, f *Forge, a CommitAsk) (*Committed, error) {
out.Files = append(out.Files, CommittedFile{Path: fc.Path, Operation: op["operation"].(string)})
}
on["files"] = ops
if err := f.sendJSON(ctx, http.MethodPost, repo+"/contents", on, &made); err != nil {
if err := f.SendJSON(ctx, http.MethodPost, repo+"/contents", on, &made); err != nil {
return nil, fmt.Errorf("the forge did not take the commit: %w", err)
}
}
@@ -274,6 +305,25 @@ func commit(ctx context.Context, f *Forge, a CommitAsk) (*Committed, error) {
return nil, fmt.Errorf("the forge answered the commit on %s without saying the commit it made: read the branch (gitea_list_branches) before trying again", a.Branch)
}
out.Commit = made.Commit.SHA
read := out.Parent
parent, err := f.parentOf(ctx, repo, made.Commit)
if err != nil {
return nil, fmt.Errorf("the forge made %s on %s, but which commit it went on could not be read: %w", out.Commit, a.Branch, err)
}
out.Parent = parent
if parent != read {
moved := "branch " + a.Branch
if out.Created {
moved = "base " + out.Base
}
how := "each file it changes was checked against " + short(read) + " by its blob, so none of them changed in between"
if a.Patch != "" {
how = "the forge applied the patch there, and it refuses a patch whose context changed"
}
out.Said = append(out.Said, fmt.Sprintf("the %s moved from %s to %s between the read and the commit, so the "+
"commit went on %s: %s. What else changed in between: gitea_compare %s...%s",
moved, short(read), short(parent), short(parent), how, short(read), short(parent)))
}
// Evidence over assertion: the branch is read back, and it must be at the commit the forge said.
after, err := f.branch(ctx, repo, a.Branch)
@@ -294,6 +344,30 @@ func commit(ctx context.Context, f *Forge, a CommitAsk) (*Committed, error) {
return out, nil
}
// madeCommit is the commit the forge answers a write with.
type madeCommit struct {
SHA string `json:"sha"`
Parents []struct {
SHA string `json:"sha"`
} `json:"parents"`
}
// parentOf is the commit the forge made its commit on: from its answer, or read from the commit when the
// answer does not say it.
func (f *Forge) parentOf(ctx context.Context, repo string, made madeCommit) (string, error) {
if len(made.Parents) == 0 {
var read madeCommit
if err := f.GetJSON(ctx, repo+"/git/commits/"+url.PathEscape(made.SHA)+"?stat=false&verification=false&files=false", &read); err != nil {
return "", err
}
made = read
}
if len(made.Parents) != 1 || made.Parents[0].SHA == "" {
return "", fmt.Errorf("the forge says %s has %d parents, not one", made.SHA, len(made.Parents))
}
return made.Parents[0].SHA, nil
}
// blobAt is the blob sha of a file at a ref, "" when there is no file there.
func (f *Forge) blobAt(ctx context.Context, repo, path, ref string) (string, error) {
seg, err := segments("file path", path)
@@ -305,7 +379,7 @@ func (f *Forge) blobAt(ctx context.Context, repo, path, ref string) (string, err
SHA string `json:"sha"`
}
var list []any
body, err := f.do(ctx, http.MethodGet, repo+"/contents/"+seg+"?ref="+queryEscape(ref), nil, maxBody)
body, err := f.Do(ctx, http.MethodGet, repo+"/contents/"+seg+"?ref="+queryEscape(ref), nil, maxBody)
if err != nil {
if errors.Is(err, errNotFound) {
return "", nil
+11 -154
View File
@@ -1,156 +1,43 @@
package main
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"os"
"regexp"
"strings"
"time"
"gitea/internal/forgehttp"
)
// The forge as this bundle reaches it: its own HTTP interface on the machine, as the forge's admin account,
// which the vault delivered into a file this module owns (novox/hq ADR 0086), by basic authentication — the
// same way the npm-registry bundle beside it does. The password is read when a call is made and never logged,
// answered or put in an error.
// The forge as this bundle reaches it: its own HTTP interface on the machine, as the forge's admin account
// (internal/forgehttp, shared with the npm-registry bundle beside it).
// Forge is the forge's HTTP interface.
type Forge struct {
URL string // e.g. http://127.0.0.1:3000
User string
PasswordFile string
HTTP *http.Client
forgehttp.Client
}
// ForgeFromEnv is the forge as the manifest hands it to this bundle.
func ForgeFromEnv() (*Forge, error) {
f := &Forge{
URL: strings.TrimRight(os.Getenv("MESH_GITEA_URL"), "/"),
User: os.Getenv("MESH_GITEA_ADMIN_USER"),
PasswordFile: os.Getenv("MESH_GITEA_ADMIN_PASSWORD_FILE"),
}
var missing []string
for _, kv := range [][2]string{{"MESH_GITEA_URL", f.URL}, {"MESH_GITEA_ADMIN_USER", f.User},
{"MESH_GITEA_ADMIN_PASSWORD_FILE", f.PasswordFile}} {
if kv[1] == "" {
missing = append(missing, kv[0])
}
}
c, missing := forgehttp.FromEnv()
if len(missing) > 0 {
return nil, fmt.Errorf("the forge cannot be reached: %s not set by the manifest", strings.Join(missing, ", "))
}
return f, nil
c.Timeout = deadline
return &Forge{Client: c}, nil
}
// errNotFound is the forge answering 404.
var errNotFound = errors.New("not found")
// statusError is an answer the forge gave that is not a success, said without the request's credential.
type statusError struct {
method, path string
status int
said string
}
func (e *statusError) Error() string {
return fmt.Sprintf("the forge answered %d to %s %s: %s", e.status, e.method, e.path, e.said)
}
func (f *Forge) client() *http.Client {
if f.HTTP != nil {
return f.HTTP
}
return &http.Client{Timeout: 50 * time.Second}
}
func (f *Forge) password() (string, error) {
raw, err := os.ReadFile(f.PasswordFile)
if err != nil {
// The path is the manifest's, not a secret; the content never appears.
return "", fmt.Errorf("the forge's admin password cannot be read from its file: %w", err)
}
return strings.TrimSpace(string(raw)), nil
}
var errNotFound = forgehttp.ErrNotFound
// maxBody is the largest JSON answer read from the forge; maxArchive the largest archive code search reads.
const (
maxBody = 32 << 20
maxBody = forgehttp.MaxBody
maxArchive = 64 << 20
)
// do sends one request and answers the body, read up to limit; 404 is errNotFound. A body the limit cut is
// refused, never read as if it were whole.
func (f *Forge) do(ctx context.Context, method, path string, payload any, limit int64) ([]byte, error) {
password, err := f.password()
if err != nil {
return nil, err
}
var body io.Reader
if payload != nil {
raw, err := json.Marshal(payload)
if err != nil {
return nil, err
}
body = bytes.NewReader(raw)
}
req, err := http.NewRequestWithContext(ctx, method, f.URL+path, body)
if err != nil {
return nil, err
}
req.SetBasicAuth(f.User, password)
req.Header.Set("Accept", "application/json")
if payload != nil {
req.Header.Set("Content-Type", "application/json")
}
res, err := f.client().Do(req)
if err != nil {
return nil, fmt.Errorf("the forge did not answer %s %s: %w", method, path, err)
}
defer res.Body.Close()
got, err := io.ReadAll(io.LimitReader(res.Body, limit+1))
if err != nil {
return nil, err
}
if int64(len(got)) > limit {
return nil, fmt.Errorf("the forge's answer to %s %s is larger than %d MiB and was not read whole", method, path, limit>>20)
}
switch {
case res.StatusCode == http.StatusNotFound:
return nil, fmt.Errorf("%s %s: %w", method, path, errNotFound)
case res.StatusCode >= 300:
said := strings.TrimSpace(string(got))
if len(said) > 300 {
said = said[:300] + "…"
}
return nil, &statusError{method, path, res.StatusCode, said}
}
return got, nil
}
func (f *Forge) getJSON(ctx context.Context, path string, into any) error {
return f.sendJSON(ctx, http.MethodGet, path, nil, into)
}
func (f *Forge) sendJSON(ctx context.Context, method, path string, payload, into any) error {
body, err := f.do(ctx, method, path, payload, maxBody)
if err != nil {
return err
}
if into == nil {
return nil
}
if err := json.Unmarshal(body, into); err != nil {
return fmt.Errorf("the forge's answer to %s %s is not readable: %w", method, path, err)
}
return nil
}
// Refused is a refusal of policy or of an argument, in the words the caller sees.
type Refused struct{ msg string }
@@ -204,36 +91,6 @@ func (p Protection) name() string {
return p.BranchName
}
// ruleCovers is whether a protection rule's name — a branch, or a glob of them (`*`, `**`, `?`) — covers a
// branch: the TypeScript bundle's ruleCovers (guard.ts), which gitea_api's file writes are judged by.
func ruleCovers(rule, branch string) bool {
if rule == branch {
return true
}
if !strings.ContainsAny(rule, "*?[") {
return false
}
var re strings.Builder
re.WriteString("^")
for i := 0; i < len(rule); i++ {
c := rule[i]
switch {
case c == '*' && i+1 < len(rule) && rule[i+1] == '*':
re.WriteString(".*")
i++
case c == '*':
re.WriteString("[^/]*")
case c == '?':
re.WriteString("[^/]")
default:
re.WriteString(regexp.QuoteMeta(string(c)))
}
}
re.WriteString("$")
ok, err := regexp.MatchString(re.String(), branch)
return err == nil && ok
}
// Branch is one branch with the commit it points at.
type Branch struct {
Name string `json:"name"`
@@ -249,7 +106,7 @@ func (f *Forge) branch(ctx context.Context, repo, name string) (*Branch, error)
return nil, err
}
var b Branch
if err := f.getJSON(ctx, repo+"/branches/"+seg, &b); err != nil {
if err := f.GetJSON(ctx, repo+"/branches/"+seg, &b); err != nil {
if errors.Is(err, errNotFound) {
return nil, nil
}
+161 -3
View File
@@ -13,9 +13,12 @@ import (
"net/http/httptest"
"os"
"path/filepath"
"regexp"
"strings"
"sync"
"testing"
"gitea/internal/forgehttp"
)
// fakeForge is the forge's HTTP interface as far as these tools use it.
@@ -37,6 +40,16 @@ type fakeForge struct {
comments map[string][]map[string]any
commits []map[string]any
dir []map[string]any
// moves: a branch moved by someone else right after it is first read — to a commit, with its tree.
moves map[string]move
noParents bool // the forge's answer to a write does not say the parents
parentsBroken bool // and GET git/commits fails
parents map[string]string // commit -> parent, as GET git/commits says it
}
type move struct {
to string
tree map[string]string
}
func newFake(t *testing.T) *fakeForge {
@@ -62,7 +75,7 @@ func (f *fakeForge) serve() *Forge {
}
srv := httptest.NewServer(http.HandlerFunc(f.handle))
f.t.Cleanup(srv.Close)
return &Forge{URL: srv.URL, User: "mesh-admin", PasswordFile: pw}
return &Forge{Client: forgehttp.Client{URL: srv.URL, User: "mesh-admin", PasswordFile: pw}}
}
func (f *fakeForge) handle(w http.ResponseWriter, r *http.Request) {
@@ -92,6 +105,11 @@ func (f *fakeForge) handle(w http.ResponseWriter, r *http.Request) {
return
}
reply(map[string]any{"name": name, "commit": map[string]any{"id": c}})
if m, ok := f.moves[name]; ok {
delete(f.moves, name)
f.trees[m.to] = m.tree
f.branches[name] = m.to
}
case r.Method == "GET" && (path == repo+"/contents" || path == repo+"/contents/docs"):
reply(f.dir)
case r.Method == "GET" && strings.HasPrefix(path, repo+"/contents/"):
@@ -148,19 +166,28 @@ func (f *fakeForge) handle(w http.ResponseWriter, r *http.Request) {
if !f.staleRead {
f.branches[target] = made
}
reply(map[string]any{"commit": map[string]any{"sha": made}})
reply(map[string]any{"commit": f.made(made, from)})
case r.Method == "POST" && path == repo+"/diffpatch":
var body map[string]any
_ = json.NewDecoder(r.Body).Decode(&body)
f.patches = append(f.patches, body)
f.next++
made := fmt.Sprintf("p%d", f.next)
from := f.branches[body["branch"].(string)]
target := body["branch"].(string)
if nb, ok := body["new_branch"].(string); ok && nb != "" {
target = nb
}
f.branches[target] = made
reply(map[string]any{"commit": map[string]any{"sha": made}})
reply(map[string]any{"commit": f.made(made, from)})
case r.Method == "GET" && strings.HasPrefix(path, repo+"/git/commits/"):
sha := strings.TrimPrefix(path, repo+"/git/commits/")
p, ok := f.parents[sha]
if !ok || f.parentsBroken {
http.NotFound(w, r)
return
}
reply(map[string]any{"sha": sha, "parents": []any{map[string]any{"sha": p}}})
case r.Method == "GET" && path == repo+"/commits":
f.writes = append(f.writes, map[string]any{"query": r.URL.RawQuery})
reply(f.commits)
@@ -180,6 +207,18 @@ func (f *fakeForge) handle(w http.ResponseWriter, r *http.Request) {
}
}
// made is the forge's answer about a commit it made on a parent, with the parents unless noParents.
func (f *fakeForge) made(sha, parent string) map[string]any {
if f.parents == nil {
f.parents = map[string]string{}
}
f.parents[sha] = parent
if f.noParents {
return map[string]any{"sha": sha}
}
return map[string]any{"sha": sha, "parents": []any{map[string]any{"sha": parent}}}
}
func filesArg(files ...map[string]any) []any {
out := make([]any, len(files))
for i, f := range files {
@@ -367,6 +406,13 @@ func TestRuleCoversIsTheGuardsRule(t *testing.T) {
{"main", "main", true}, {"main", "mainx", false}, {"release/*", "release/1.0", true},
{"release/*", "release/a/b", false}, {"release/**", "release/a/b", true}, {"v?", "v1", true},
{"a.b", "axb", false},
// As the forge decides it (novox/hq issue 539): classes, alternatives, escapes, a plain name in any case.
{"release-[ab]", "release-a", true}, {"release-[ab]", "release-c", false}, {"release-[!ab]", "release-c", true},
{"v[0-9].x", "v7.x", true}, {"v[0-9].x", "vA.x", false}, {"{main,master}", "master", true},
{"{main,master}", "mainline", false}, {"rel/{a,b}/*", "rel/b/1", true}, {"rel/{a,b}/*", "rel/c/1", false},
{"Main", "main", true}, {"main", "MAIN", true}, {"Feat/*", "feat/x", false}, {`a\*`, "a*", true},
{`a\*`, "ab", false}, {"a+b", "a+b", true}, {"a+b", "aab", false}, {"{open", "{open", true},
{"[open", "[open", true}, {"[open", "o", false},
} {
if got := ruleCovers(c.rule, c.branch); got != c.want {
t.Errorf("%s covers %s: %v", c.rule, c.branch, got)
@@ -581,3 +627,115 @@ func TestToolsAreTheOnesTheReadmeNamesAndTheManifestBuilds(t *testing.T) {
t.Error("the manifest builds no cmd/forge bundle")
}
}
// novox/hq issue 539: the forge takes a branch, never a commit to build on, so the answer's parent is the one
// the forge made the commit on, and a branch that moves between the read and the commit is said.
func TestAnExistingBranchMovedUnderTheCommitIsSaidAndItsParentIsTheOneTheForgeUsed(t *testing.T) {
f := newFake(t)
f.moves = map[string]move{"fix/1-old": {to: "c2", tree: map[string]string{"README.md": "hello\n", "docs/a.md": "a on fix\n", "other.md": "x\n"}}}
got, err := commit(context.Background(), f.serve(), ask(t, map[string]any{"branch": "fix/1-old",
"files": filesArg(map[string]any{"path": "docs/a.md", "content": "a fixed\n"})}))
if err != nil {
t.Fatal(err)
}
if got.Parent != "c2" || f.branches["fix/1-old"] != got.Commit {
t.Fatalf("parent %s, want c2 (the commit the forge built on): %+v", got.Parent, got)
}
said := strings.Join(got.Said, " ")
if !strings.Contains(said, "moved from c1 to c2") || !strings.Contains(said, "after c2") {
t.Fatalf("the move was not said: %v", got.Said)
}
}
func TestAnExistingBranchsFilesAreJudgedAtTheCommitReadNotAtWhereTheBranchMovedTo(t *testing.T) {
f := newFake(t)
// Someone else changes docs/a.md on the branch between the read and the commit: that change is not
// overwritten by a blob read at the branch's new head.
f.moves = map[string]move{"fix/1-old": {to: "c2", tree: map[string]string{"README.md": "hello\n", "docs/a.md": "theirs\n"}}}
_, err := commit(context.Background(), f.serve(), ask(t, map[string]any{"branch": "fix/1-old",
"files": filesArg(map[string]any{"path": "docs/a.md", "content": "mine\n"})}))
if err == nil || !strings.Contains(err.Error(), "sha does not match") {
t.Fatalf("a file changed under the commit was overwritten: %v", err)
}
if f.trees["c2"]["docs/a.md"] != "theirs\n" || f.branches["fix/1-old"] != "c2" {
t.Fatalf("the branch was changed: %v", f.branches)
}
}
func TestAPatchOnABaseThatMovedNamesTheCommitTheForgeAppliedItTo(t *testing.T) {
f := newFake(t)
f.moves = map[string]move{"main": {to: "c9", tree: map[string]string{"README.md": "hello\n"}}}
patch := "diff --git a/README.md b/README.md\n--- a/README.md\n+++ b/README.md\n@@ -1 +1 @@\n-hello\n+hi\n"
got, err := commit(context.Background(), f.serve(), ask(t, map[string]any{"branch": "fix/6", "patch": patch}))
if err != nil {
t.Fatal(err)
}
if got.Parent != "c9" || !strings.Contains(strings.Join(got.Said, " "), "base main moved from c0 to c9") {
t.Fatalf("%+v", got)
}
}
func TestAParentTheAnswerDoesNotSayIsReadFromTheCommit(t *testing.T) {
f := newFake(t)
f.noParents = true
got, err := commit(context.Background(), f.serve(), ask(t, map[string]any{"branch": "fix/7",
"files": filesArg(map[string]any{"path": "x", "content": "x"})}))
if err != nil || got.Parent != "c0" {
t.Fatalf("%v %+v", err, got)
}
// A parent that cannot be read is a failure, said with the commit that was made.
f = newFake(t)
f.noParents, f.parentsBroken = true, true
_, err = commit(context.Background(), f.serve(), ask(t, map[string]any{"branch": "fix/8",
"files": filesArg(map[string]any{"path": "x", "content": "x"})}))
if err == nil || !strings.Contains(err.Error(), "the forge made n1 on fix/8, but which commit it went on could not be read") {
t.Fatalf("%v", err)
}
}
func TestACommitLargerThanItsLimitsIsRefusedBeforeAnythingIsSent(t *testing.T) {
big := strings.Repeat("x", maxFileBytes+1)
quarter := strings.Repeat("y", maxFileBytes)
many := filesArg()
for i := 0; i < maxCommitBytes/maxFileBytes+1; i++ {
many = append(many, map[string]any{"path": fmt.Sprintf("f%d", i), "content": quarter})
}
for name, args := range map[string]map[string]any{
"one file": {"files": filesArg(map[string]any{"path": "x", "content": big})},
"one file b64": {"files": filesArg(map[string]any{"path": "x", "content_base64": base64.StdEncoding.EncodeToString([]byte(big))})},
"the patch": {"patch": "diff --git a/x b/x\n" + big},
"all files": {"files": many},
} {
full := map[string]any{"owner": "acme", "repo": "app", "message": "m", "branch": "b"}
for k, v := range args {
full[k] = v
}
_, err := readCommitAsk(full)
if err == nil || !strings.Contains(err.Error(), "MiB") || !strings.Contains(err.Error(), "Nothing was sent") {
t.Errorf("%s: %v", name, err)
}
}
// At the limit is taken.
if _, err := readCommitAsk(map[string]any{"owner": "acme", "repo": "app", "message": "m", "branch": "b",
"files": filesArg(map[string]any{"path": "x", "content_base64": base64.StdEncoding.EncodeToString([]byte(quarter))})}); err != nil {
t.Errorf("a file at the limit was refused: %v", err)
}
}
func TestCodeSearchStopsAtItsDeadlineAndSaysSo(t *testing.T) {
gz := archive(t, map[string]string{"a.md": "forge\n", "b.md": "forge\n"})
ctx, cancel := context.WithCancel(context.Background())
cancel()
out := &Found{Matches: []Match{}}
if err := searchArchive(ctx, gz, "", regexp.MustCompile("forge"), 100, out); err != nil {
t.Fatal(err)
}
if !out.Stopped || out.Files != 0 {
t.Fatalf("a search past its deadline went on: %+v", out)
}
// Within its deadline, the same archive is searched whole.
out = &Found{Matches: []Match{}}
if err := searchArchive(context.Background(), gz, "", regexp.MustCompile("forge"), 100, out); err != nil || out.Stopped || len(out.Matches) != 2 {
t.Fatalf("%v %+v", err, out)
}
}
+3 -1
View File
@@ -53,7 +53,9 @@ func Tools(forge func() (*Forge, error)) []stdio.Tool {
"independent review approved and whose required checks passed (gitea_merge_pull_request). Carries " +
"either files — each {path, content (text) | content_base64 | delete: true}, created or updated as " +
"the branch holds them — or patch, a unified diff as git diff writes it, which the forge applies. " +
"Answers the commit it made and its parent, read back from the branch. Replaces git commit and git " +
"At most 200 files, 4 MiB each and 16 MiB together, or a 4 MiB patch. Answers the commit it made and " +
"the parent the forge made it on, read back from the branch, and says when the branch moved under " +
"it. Replaces git commit and git " +
"push of a feature branch. (a)",
Input: map[string]any{
"type": "object",
+6 -6
View File
@@ -72,7 +72,7 @@ func history(ctx context.Context, f *Forge, owner, name, ref, path string, limit
q.Set("verification", "false")
q.Set("files", "false")
var got []forgeCommit
if err := f.getJSON(ctx, repo+"/commits?"+q.Encode(), &got); err != nil {
if err := f.GetJSON(ctx, repo+"/commits?"+q.Encode(), &got); err != nil {
return nil, err
}
h := &History{Repository: owner + "/" + name, Ref: ref, Path: path, Page: page, Commits: []CommitLine{}, More: len(got) >= limit}
@@ -121,7 +121,7 @@ func compare(ctx context.Context, f *Forge, owner, name, base, head string) (*Co
Total int `json:"total_commits"`
Commits []forgeCommit `json:"commits"`
}
if err := f.getJSON(ctx, repo+"/compare/"+b+"..."+h, &got); err != nil {
if err := f.GetJSON(ctx, repo+"/compare/"+b+"..."+h, &got); err != nil {
return nil, err
}
c := &Comparison{Repository: owner + "/" + name, Base: base, Head: head, Total: got.Total, Commits: []CommitLine{}, Files: []FileLine{}}
@@ -195,7 +195,7 @@ func reviews(ctx context.Context, f *Forge, owner, name string, number int) (*Re
SHA string `json:"sha"`
} `json:"head"`
}
if err := f.getJSON(ctx, pr, &pull); err != nil {
if err := f.GetJSON(ctx, pr, &pull); err != nil {
if errors.Is(err, errNotFound) {
return nil, refuse("%s/%s has no pull request %d", owner, name, number)
}
@@ -218,7 +218,7 @@ func reviews(ctx context.Context, f *Forge, owner, name string, number int) (*Re
var all []forgeReview
for p := 1; ; p++ {
var batch []forgeReview
if err := f.getJSON(ctx, fmt.Sprintf("%s/reviews?limit=%d&page=%d", pr, page, p), &batch); err != nil {
if err := f.GetJSON(ctx, fmt.Sprintf("%s/reviews?limit=%d&page=%d", pr, page, p), &batch); err != nil {
return nil, err
}
all = append(all, batch...)
@@ -240,7 +240,7 @@ func reviews(ctx context.Context, f *Forge, owner, name string, number int) (*Re
CommitID string `json:"commit_id"`
Resolver *forgeUser `json:"resolver"`
}
if err := f.getJSON(ctx, fmt.Sprintf("%s/reviews/%d/comments", pr, r.ID), &cs); err != nil {
if err := f.GetJSON(ctx, fmt.Sprintf("%s/reviews/%d/comments", pr, r.ID), &cs); err != nil {
return nil, fmt.Errorf("review %d's line comments could not be read: %w", r.ID, err)
}
for _, c := range cs {
@@ -292,7 +292,7 @@ func listDir(ctx context.Context, f *Forge, owner, name, path, ref string) (*Lis
if ref != "" {
at += "?ref=" + queryEscape(ref)
}
body, err := f.do(ctx, "GET", at, nil, maxBody)
body, err := f.Do(ctx, "GET", at, nil, maxBody)
if err != nil {
if errors.Is(err, errNotFound) {
return nil, refuse("%s/%s has no %q at %s", owner, name, path, orDefault(ref))
+116
View File
@@ -0,0 +1,116 @@
package main
import (
"errors"
"regexp"
"strings"
)
// ruleCovers is whether a protection rule's name covers a branch, as the forge itself decides it (Gitea 1.27,
// models/git/protected_branch.go Match and modules/glob, novox/hq issue 539): a name without a glob character
// covers the branch of that name in any case; any other name is a glob with `/` as its separator — `*` any
// run within a segment, `**` any run across them, `?` one character of a segment, `[abc]`, `[a-z]` and `[!abc]`
// one character, `{a,b}` either, `\` the next character itself — matched case-sensitively; a glob the forge
// cannot compile covers its own text exactly. The TypeScript bundle's ruleCovers (guard.ts) is the same rule.
func ruleCovers(rule, branch string) bool {
if !strings.ContainsAny(rule, `*?\[]{}`) {
return strings.EqualFold(rule, branch)
}
re, err := globRegexp(rule)
if err != nil {
return rule == branch
}
return re.MatchString(branch)
}
// globRegexp is the forge's glob as a regular expression, compiled as modules/glob compiles it with `/` as
// the separator.
func globRegexp(glob string) (*regexp.Regexp, error) {
g := []rune(glob)
var b strings.Builder
pos := 0
var compile func(sub bool) error
chars := func() error {
b.WriteByte('[')
if pos < len(g) && g[pos] == '!' {
pos++
b.WriteByte('^')
}
for pos < len(g) {
c := g[pos]
pos++
switch {
case c == ']':
b.WriteByte(']')
return nil
case c == '\\':
if pos >= len(g) {
return errors.New("unterminated character class escape")
}
b.WriteByte('\\')
b.WriteRune(g[pos])
pos++
default:
b.WriteRune(c)
}
}
return errors.New("unterminated character class")
}
compile = func(sub bool) error {
for pos < len(g) {
c := g[pos]
pos++
if sub && c == '}' {
b.WriteByte(')')
return nil
}
switch c {
case '*':
if pos < len(g) && g[pos] == '*' {
pos++
b.WriteString(".*")
} else {
b.WriteString("[^/]*")
}
case '?':
b.WriteString("[^/]")
case '+':
b.WriteString(`\+`)
case '[':
if err := chars(); err != nil {
return err
}
case '{':
b.WriteByte('(')
if err := compile(true); err != nil {
return err
}
case ',':
if sub {
b.WriteByte('|')
} else {
b.WriteByte(',')
}
case '\\':
if pos >= len(g) {
return errors.New("no character to escape")
}
b.WriteByte('\\')
b.WriteRune(g[pos])
pos++
case '.', '^', '$', '(', ')', '|':
b.WriteByte('\\')
b.WriteRune(c)
default:
b.WriteRune(c)
}
}
return nil
}
b.WriteByte('^')
if err := compile(false); err != nil {
return nil, err
}
b.WriteByte('$')
return regexp.Compile(b.String())
}
+42 -7
View File
@@ -32,6 +32,7 @@ type Found struct {
Files int `json:"files_searched"`
Skipped []string `json:"skipped,omitempty"`
Truncated bool `json:"truncated"`
Stopped bool `json:"stopped_at_deadline,omitempty"`
Said []string `json:"said"`
}
@@ -69,7 +70,7 @@ func searchCode(ctx context.Context, f *Forge, a SearchAsk) (*Found, error) {
ref := a.Ref
if ref == "" {
var r Repo
if err := f.getJSON(ctx, repo, &r); err != nil {
if err := f.GetJSON(ctx, repo, &r); err != nil {
return nil, err
}
if r.Empty || r.DefaultBranch == "" {
@@ -87,7 +88,7 @@ func searchCode(ctx context.Context, f *Forge, a SearchAsk) (*Found, error) {
return nil, err
}
}
body, err := f.do(ctx, "GET", repo+"/archive/"+seg+".tar.gz", nil, maxArchive)
body, err := f.Do(ctx, "GET", repo+"/archive/"+seg+".tar.gz", nil, maxArchive)
if err != nil {
if errors.Is(err, errNotFound) {
return nil, refuse("%s/%s has no ref %s", a.Owner, a.Repo, ref)
@@ -95,11 +96,15 @@ func searchCode(ctx context.Context, f *Forge, a SearchAsk) (*Found, error) {
return nil, err
}
out := &Found{Repository: a.Owner + "/" + a.Repo, Ref: ref, Query: a.Query, Matches: []Match{}}
if err := searchArchive(body, prefix, re, a.Max, out); err != nil {
if err := searchArchive(ctx, body, prefix, re, a.Max, out); err != nil {
return nil, fmt.Errorf("the archive of %s at %s could not be read: %w", out.Repository, ref, err)
}
said := fmt.Sprintf("%d matches in %d files searched", len(out.Matches), out.Files)
if out.Truncated {
switch {
case out.Stopped:
said = fmt.Sprintf("the search stopped at its deadline of %s after %d files, with %d matches: the rest of the "+
"archive was not searched. Narrow with path", deadline, out.Files, len(out.Matches))
case out.Truncated:
said = fmt.Sprintf("the first %d matches, and more were found: narrow with path or the query", len(out.Matches))
}
out.Said = append(out.Said, said)
@@ -110,18 +115,32 @@ func searchCode(ctx context.Context, f *Forge, a SearchAsk) (*Found, error) {
}
// searchArchive reads a gzipped tar the forge made — every path under one top directory — and collects the
// lines that match, from text files under the prefix.
func searchArchive(gz []byte, prefix string, re *regexp.Regexp, max int, out *Found) error {
// lines that match, from text files under the prefix. It stops at the call's deadline (novox/hq issue 539):
// what it found by then is kept, and out.Stopped says the rest was not searched.
func searchArchive(ctx context.Context, gz []byte, prefix string, re *regexp.Regexp, max int, out *Found) error {
zr, err := gzip.NewReader(bytes.NewReader(gz))
if err != nil {
return err
}
tr := tar.NewReader(zr)
tr := tar.NewReader(ctxReader{ctx, zr})
stopped := func(err error) bool {
if ctx.Err() != nil && (err == nil || errors.Is(err, ctx.Err())) {
out.Stopped = true
return true
}
return false
}
for {
if stopped(nil) {
return nil
}
h, err := tr.Next()
if err == io.EOF {
return nil
}
if stopped(err) {
return nil
}
if err != nil {
return err
}
@@ -140,6 +159,9 @@ func searchArchive(gz []byte, prefix string, re *regexp.Regexp, max int, out *Fo
continue
}
content, err := io.ReadAll(io.LimitReader(tr, maxFileSearched+1))
if stopped(err) {
return nil
}
if err != nil {
return err
}
@@ -169,3 +191,16 @@ func searchArchive(gz []byte, prefix string, re *regexp.Regexp, max int, out *Fo
}
}
}
// ctxReader is a reader that ends when its context does: decompressing is the long part of a search.
type ctxReader struct {
ctx context.Context
r io.Reader
}
func (c ctxReader) Read(p []byte) (int, error) {
if err := c.ctx.Err(); err != nil {
return 0, err
}
return c.r.Read(p)
}